<commit_before>package client

import (
	"fmt"
	"os"
	"time"

	"golang.org/x/net/context"

	"google.golang.org/grpc"
	"google.golang.org/grpc/keepalive"
	"google.golang.org/grpc/metadata"

	types "github.com/gogo/protobuf/types"
	log "github.com/sirupsen/logrus"

	"github.com/pachyderm/pachyderm/src/client/auth"
	"github.com/pachyderm/pachyderm/src/client/deploy"
	"github.com/pachyderm/pachyderm/src/client/enterprise"
	"github.com/pachyderm/pachyderm/src/client/health"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/config"
	"github.com/pachyderm/pachyderm/src/client/pkg/grpcutil"
	"github.com/pachyderm/pachyderm/src/client/pps"
)

const (
	// MaxListItemsLog specifies the maximum number of items we log in response to a List* API
	MaxListItemsLog = 10
	// StorageSecretName is the name of the Kubernetes secret in which
	// storage credentials are stored.
	StorageSecretName = "pachyderm-storage-secret"
)

// PfsAPIClient is an alias for pfs.APIClient.
type PfsAPIClient pfs.APIClient

// PpsAPIClient is an alias for pps.APIClient.
type PpsAPIClient pps.APIClient

// ObjectAPIClient is an alias for pfs.ObjectAPIClient
type ObjectAPIClient pfs.ObjectAPIClient

// AuthAPIClient is an alias of auth.APIClient
type AuthAPIClient auth.APIClient

// DeployAPIClient is an alias of auth.APIClient
type DeployAPIClient deploy.APIClient

// An APIClient is a wrapper around pfs, pps and block APIClients.
type APIClient struct {
	PfsAPIClient
	PpsAPIClient
	ObjectAPIClient
	AuthAPIClient
	DeployAPIClient
	Enterprise enterprise.APIClient // not embedded--method name conflicts with AuthAPIClient

	// addr is a "host:port" string pointing at a pachd endpoint
	addr string

	// clientConn is a cached grpc connection to 'addr'
	clientConn *grpc.ClientConn

	// healthClient is a cached healthcheck client connected to 'addr'
	healthClient health.HealthClient

	// streamSemaphore limits the number of concurrent message streams between
	// this client and pachd
	streamSemaphore chan struct{}

	// metricsUserID is an identifier that is included in usage metrics sent to
	// Pachyderm Inc. and is used to count the number of unique Pachyderm users.
	// If unset, no usage metrics are sent back to Pachyderm Inc.
	metricsUserID string

	// metricsPrefix is used to send information from this client to Pachyderm Inc
	// for usage metrics
	metricsPrefix string

	// authenticationToken is an identifier that authenticates the caller in case
	// they want to access privileged data
	authenticationToken string

	// The context used in requests, can be set with WithCtx
	ctx context.Context
}

// GetAddress returns the pachd host:port with which 'c' is communicating. If
// 'c' was created using NewInCluster or NewOnUserMachine then this is how the
// address may be retrieved from the environment.
func (c *APIClient) GetAddress() string {
	return c.addr
}

// DefaultMaxConcurrentStreams defines the max number of Putfiles or Getfiles happening simultaneously
const DefaultMaxConcurrentStreams uint = 100

// NewFromAddressWithConcurrency constructs a new APIClient and sets the max
// concurrency of streaming requests (GetFile / PutFile)
func NewFromAddressWithConcurrency(addr string, maxConcurrentStreams uint) (*APIClient, error) {
	c := &APIClient{
		addr:            addr,
		streamSemaphore: make(chan struct{}, maxConcurrentStreams),
	}
	if err := c.connect(); err != nil {
		return nil, err
	}
	return c, nil
}

// NewFromAddress constructs a new APIClient for the server at addr.
func NewFromAddress(addr string) (*APIClient, error) {
	return NewFromAddressWithConcurrency(addr, DefaultMaxConcurrentStreams)
}

// GetAddressFromUserMachine interprets the Pachyderm config in 'cfg' in the
// context of local environment variables and returns a "host:port" string
// pointing at a Pachd target.
func GetAddressFromUserMachine(cfg *config.Config) string {
	address := "0.0.0.0:30650"
	if cfg != nil && cfg.V1 != nil && cfg.V1.PachdAddress != "" {
		address = cfg.V1.PachdAddress
	}
	// ADDRESS environment variable (shell-local) overrides global config
	if envAddr := os.Getenv("ADDRESS"); envAddr != "" {
		address = envAddr
	}
	return address
}

// NewOnUserMachine constructs a new APIClient using env vars that may be set
// on a user's machine (i.e. ADDRESS), as well as $HOME/.pachyderm/config if it
// exists. This is primarily intended to be used with the pachctl binary, but
// may also be useful in tests.
//
// TODO(msteffen) this logic is fairly linux/unix specific, and makes the
// pachyderm client library incompatible with Windows. We may want to move this
// (and similar) logic into src/server and have it call a NewFromOptions()
// constructor.
func NewOnUserMachine(reportMetrics bool, prefix string) (*APIClient, error) {
	return NewOnUserMachineWithConcurrency(reportMetrics, prefix, DefaultMaxConcurrentStreams)
}

// NewOnUserMachineWithConcurrency is identical to NewOnUserMachine, but
// explicitly sets a limit on the number of RPC streams that may be open
// simultaneously
func NewOnUserMachineWithConcurrency(reportMetrics bool, prefix string, maxConcurrentStreams uint) (*APIClient, error) {
	cfg, err := config.Read()
	if err != nil {
		// metrics errors are non fatal
		log.Warningf("error loading user config from ~/.pachderm/config: %v", err)
	}

	// create new pachctl client
	client, err := NewFromAddress(GetAddressFromUserMachine(cfg))
	if err != nil {
		return nil, err
	}

	// Add metrics info & authentication token
	client.metricsPrefix = prefix
	if cfg.UserID != "" && reportMetrics {
		client.metricsUserID = cfg.UserID
	}
	if cfg.V1 != nil && cfg.V1.SessionToken != "" {
		client.authenticationToken = cfg.V1.SessionToken
	}
	return client, nil
}

// NewInCluster constructs a new APIClient using env vars that Kubernetes creates.
// This should be used to access Pachyderm from within a Kubernetes cluster
// with Pachyderm running on it.
func NewInCluster() (*APIClient, error) {
	if addr := os.Getenv("PACHD_PORT_650_TCP_ADDR"); addr != "" {
		return NewFromAddress(fmt.Sprintf("%v:650", addr))
	}
	return nil, fmt.Errorf("PACHD_PORT_650_TCP_ADDR not set")
}

// Close the connection to gRPC
func (c *APIClient) Close() error {
	return c.clientConn.Close()
}

// DeleteAll deletes everything in the cluster.
// Use with caution, there is no undo.
func (c APIClient) DeleteAll() error {
	if _, err := c.AuthAPIClient.Deactivate(
		c.Ctx(),
		&auth.DeactivateRequest{},
	); err != nil && !auth.IsNotActivatedError(err) {
		return grpcutil.ScrubGRPC(err)
	}
	if _, err := c.PpsAPIClient.DeleteAll(
		c.Ctx(),
		&types.Empty{},
	); err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	if _, err := c.PfsAPIClient.DeleteAll(
		c.Ctx(),
		&types.Empty{},
	); err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	return nil
}

// SetMaxConcurrentStreams Sets the maximum number of concurrent streams the
// client can have. It is not safe to call this operations while operations are
// outstanding.
func (c APIClient) SetMaxConcurrentStreams(n int) {
	c.streamSemaphore = make(chan struct{}, n)
}

// EtcdDialOptions is a helper returning a slice of grpc.Dial options
// such that grpc.Dial() is synchronous: the call doesn't return until
// the connection has been established and it's safe to send RPCs
func EtcdDialOptions() []grpc.DialOption {
	return []grpc.DialOption{
		// Don't return from Dial() until the connection has been established
		grpc.WithBlock(),

		// If no connection is established in 30s, fail the call
		grpc.WithTimeout(30 * time.Second),

		grpc.WithDefaultCallOptions(
			grpc.MaxCallRecvMsgSize(grpcutil.MaxMsgSize),
			grpc.MaxCallSendMsgSize(grpcutil.MaxMsgSize),
		),
	}
}

// PachDialOptions is a helper returning a slice of grpc.Dial options
// such that
// - TLS is disabled
// - Dial is synchronous: the call doesn't return until the connection has been
//                        established and it's safe to send RPCs
//
// This is primarily useful for Pachd and Worker clients
func PachDialOptions() []grpc.DialOption {
	return append(EtcdDialOptions(), grpc.WithInsecure())
}

func (c *APIClient) connect() error {
	keepaliveOpt := grpc.WithKeepaliveParams(keepalive.ClientParameters{
		Time:                20 * time.Second, // if 20s since last msg (any kind), ping
		Timeout:             20 * time.Second, // if no response to ping for 20s, reset
		PermitWithoutStream: true,             // send ping even if no active RPCs
	})
	dialOptions := append(PachDialOptions(), keepaliveOpt)
	clientConn, err := grpc.Dial(c.addr, dialOptions...)
	if err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	c.PfsAPIClient = pfs.NewAPIClient(clientConn)
	c.PpsAPIClient = pps.NewAPIClient(clientConn)
	c.ObjectAPIClient = pfs.NewObjectAPIClient(clientConn)
	c.AuthAPIClient = auth.NewAPIClient(clientConn)
	c.Enterprise = enterprise.NewAPIClient(clientConn)
	c.DeployAPIClient = deploy.NewAPIClient(clientConn)
	c.clientConn = clientConn
	c.healthClient = health.NewHealthClient(clientConn)
	return nil
}

// AddMetadata adds necessary metadata (including authentication credentials)
// to the context 'ctx', preserving any metadata that is present in either the
// incoming or outgoing metadata of 'ctx'.
func (c *APIClient) AddMetadata(ctx context.Context) context.Context {
	// TODO(msteffen): There are several places in this client where it's possible
	// to set per-request metadata (specifically auth tokens): client.WithCtx(),
	// client.SetAuthToken(), etc. These should be consolidated, as this API
	// doesn't make it obvious how these settings are resolved when they conflict.
	clientData := make(map[string]string)
	if c.authenticationToken != "" {
		clientData[auth.ContextTokenKey] = c.authenticationToken
	}
	// metadata API downcases all the key names
	if c.metricsUserID != "" {
		clientData["userid"] = c.metricsUserID
		clientData["prefix"] = c.metricsPrefix
	}

	// Rescue any metadata pairs already in 'ctx' (otherwise
	// metadata.NewOutgoingContext() would drop them). Note that this is similar
	// to metadata.Join(), but distinct because it discards conflicting k/v pairs
	// instead of merging them)
	incomingMD, _ := metadata.FromIncomingContext(ctx)
	outgoingMD, _ := metadata.FromOutgoingContext(ctx)
	clientMD := metadata.New(clientData)
	finalMD := make(metadata.MD) // Collect k/v pairs
	for _, md := range []metadata.MD{incomingMD, outgoingMD, clientMD} {
		for k, v := range md {
			finalMD[k] = v
		}
	}
	return metadata.NewOutgoingContext(ctx, finalMD)
}

// Ctx is a convenience function that returns adds Pachyderm authn metadata
// to context.Background().
func (c *APIClient) Ctx() context.Context {
	if c.ctx == nil {
		return c.AddMetadata(context.Background())
	}
	return c.AddMetadata(c.ctx)
}

// WithCtx returns a new APIClient that uses ctx for requests it sends. Note
// that the new APIClient will still use the authentication token and metrics
// metadata of this client, so this is only useful for propagating other
// context-associated metadata.
func (c *APIClient) WithCtx(ctx context.Context) *APIClient {
	result := *c // copy c
	result.ctx = ctx
	return &result
}

// SetAuthToken sets the authentication token that will be used for all
// API calls for this client.
func (c *APIClient) SetAuthToken(token string) {
	c.authenticationToken = token
}
<commit_msg>Adds version client to client.<commit_after>package client

import (
	"fmt"
	"os"
	"time"

	"golang.org/x/net/context"

	"google.golang.org/grpc"
	"google.golang.org/grpc/keepalive"
	"google.golang.org/grpc/metadata"

	types "github.com/gogo/protobuf/types"
	log "github.com/sirupsen/logrus"

	"github.com/pachyderm/pachyderm/src/client/auth"
	"github.com/pachyderm/pachyderm/src/client/deploy"
	"github.com/pachyderm/pachyderm/src/client/enterprise"
	"github.com/pachyderm/pachyderm/src/client/health"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/config"
	"github.com/pachyderm/pachyderm/src/client/pkg/grpcutil"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/client/version/versionpb"
)

const (
	// MaxListItemsLog specifies the maximum number of items we log in response to a List* API
	MaxListItemsLog = 10
	// StorageSecretName is the name of the Kubernetes secret in which
	// storage credentials are stored.
	StorageSecretName = "pachyderm-storage-secret"
)

// PfsAPIClient is an alias for pfs.APIClient.
type PfsAPIClient pfs.APIClient

// PpsAPIClient is an alias for pps.APIClient.
type PpsAPIClient pps.APIClient

// ObjectAPIClient is an alias for pfs.ObjectAPIClient
type ObjectAPIClient pfs.ObjectAPIClient

// AuthAPIClient is an alias of auth.APIClient
type AuthAPIClient auth.APIClient

// DeployAPIClient is an alias of auth.APIClient
type DeployAPIClient deploy.APIClient

type VersionAPIClient versionpb.APIClient

// An APIClient is a wrapper around pfs, pps and block APIClients.
type APIClient struct {
	PfsAPIClient
	PpsAPIClient
	ObjectAPIClient
	AuthAPIClient
	DeployAPIClient
	VersionAPIClient
	Enterprise enterprise.APIClient // not embedded--method name conflicts with AuthAPIClient

	// addr is a "host:port" string pointing at a pachd endpoint
	addr string

	// clientConn is a cached grpc connection to 'addr'
	clientConn *grpc.ClientConn

	// healthClient is a cached healthcheck client connected to 'addr'
	healthClient health.HealthClient

	// streamSemaphore limits the number of concurrent message streams between
	// this client and pachd
	streamSemaphore chan struct{}

	// metricsUserID is an identifier that is included in usage metrics sent to
	// Pachyderm Inc. and is used to count the number of unique Pachyderm users.
	// If unset, no usage metrics are sent back to Pachyderm Inc.
	metricsUserID string

	// metricsPrefix is used to send information from this client to Pachyderm Inc
	// for usage metrics
	metricsPrefix string

	// authenticationToken is an identifier that authenticates the caller in case
	// they want to access privileged data
	authenticationToken string

	// The context used in requests, can be set with WithCtx
	ctx context.Context
}

// GetAddress returns the pachd host:port with which 'c' is communicating. If
// 'c' was created using NewInCluster or NewOnUserMachine then this is how the
// address may be retrieved from the environment.
func (c *APIClient) GetAddress() string {
	return c.addr
}

// DefaultMaxConcurrentStreams defines the max number of Putfiles or Getfiles happening simultaneously
const DefaultMaxConcurrentStreams uint = 100

// NewFromAddressWithConcurrency constructs a new APIClient and sets the max
// concurrency of streaming requests (GetFile / PutFile)
func NewFromAddressWithConcurrency(addr string, maxConcurrentStreams uint) (*APIClient, error) {
	c := &APIClient{
		addr:            addr,
		streamSemaphore: make(chan struct{}, maxConcurrentStreams),
	}
	if err := c.connect(); err != nil {
		return nil, err
	}
	return c, nil
}

// NewFromAddress constructs a new APIClient for the server at addr.
func NewFromAddress(addr string) (*APIClient, error) {
	return NewFromAddressWithConcurrency(addr, DefaultMaxConcurrentStreams)
}

// GetAddressFromUserMachine interprets the Pachyderm config in 'cfg' in the
// context of local environment variables and returns a "host:port" string
// pointing at a Pachd target.
func GetAddressFromUserMachine(cfg *config.Config) string {
	address := "0.0.0.0:30650"
	if cfg != nil && cfg.V1 != nil && cfg.V1.PachdAddress != "" {
		address = cfg.V1.PachdAddress
	}
	// ADDRESS environment variable (shell-local) overrides global config
	if envAddr := os.Getenv("ADDRESS"); envAddr != "" {
		address = envAddr
	}
	return address
}

// NewOnUserMachine constructs a new APIClient using env vars that may be set
// on a user's machine (i.e. ADDRESS), as well as $HOME/.pachyderm/config if it
// exists. This is primarily intended to be used with the pachctl binary, but
// may also be useful in tests.
//
// TODO(msteffen) this logic is fairly linux/unix specific, and makes the
// pachyderm client library incompatible with Windows. We may want to move this
// (and similar) logic into src/server and have it call a NewFromOptions()
// constructor.
func NewOnUserMachine(reportMetrics bool, prefix string) (*APIClient, error) {
	return NewOnUserMachineWithConcurrency(reportMetrics, prefix, DefaultMaxConcurrentStreams)
}

// NewOnUserMachineWithConcurrency is identical to NewOnUserMachine, but
// explicitly sets a limit on the number of RPC streams that may be open
// simultaneously
func NewOnUserMachineWithConcurrency(reportMetrics bool, prefix string, maxConcurrentStreams uint) (*APIClient, error) {
	cfg, err := config.Read()
	if err != nil {
		// metrics errors are non fatal
		log.Warningf("error loading user config from ~/.pachderm/config: %v", err)
	}

	// create new pachctl client
	client, err := NewFromAddress(GetAddressFromUserMachine(cfg))
	if err != nil {
		return nil, err
	}

	// Add metrics info & authentication token
	client.metricsPrefix = prefix
	if cfg.UserID != "" && reportMetrics {
		client.metricsUserID = cfg.UserID
	}
	if cfg.V1 != nil && cfg.V1.SessionToken != "" {
		client.authenticationToken = cfg.V1.SessionToken
	}
	return client, nil
}

// NewInCluster constructs a new APIClient using env vars that Kubernetes creates.
// This should be used to access Pachyderm from within a Kubernetes cluster
// with Pachyderm running on it.
func NewInCluster() (*APIClient, error) {
	if addr := os.Getenv("PACHD_PORT_650_TCP_ADDR"); addr != "" {
		return NewFromAddress(fmt.Sprintf("%v:650", addr))
	}
	return nil, fmt.Errorf("PACHD_PORT_650_TCP_ADDR not set")
}

// Close the connection to gRPC
func (c *APIClient) Close() error {
	return c.clientConn.Close()
}

// DeleteAll deletes everything in the cluster.
// Use with caution, there is no undo.
func (c APIClient) DeleteAll() error {
	if _, err := c.AuthAPIClient.Deactivate(
		c.Ctx(),
		&auth.DeactivateRequest{},
	); err != nil && !auth.IsNotActivatedError(err) {
		return grpcutil.ScrubGRPC(err)
	}
	if _, err := c.PpsAPIClient.DeleteAll(
		c.Ctx(),
		&types.Empty{},
	); err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	if _, err := c.PfsAPIClient.DeleteAll(
		c.Ctx(),
		&types.Empty{},
	); err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	return nil
}

// SetMaxConcurrentStreams Sets the maximum number of concurrent streams the
// client can have. It is not safe to call this operations while operations are
// outstanding.
func (c APIClient) SetMaxConcurrentStreams(n int) {
	c.streamSemaphore = make(chan struct{}, n)
}

// EtcdDialOptions is a helper returning a slice of grpc.Dial options
// such that grpc.Dial() is synchronous: the call doesn't return until
// the connection has been established and it's safe to send RPCs
func EtcdDialOptions() []grpc.DialOption {
	return []grpc.DialOption{
		// Don't return from Dial() until the connection has been established
		grpc.WithBlock(),

		// If no connection is established in 30s, fail the call
		grpc.WithTimeout(30 * time.Second),

		grpc.WithDefaultCallOptions(
			grpc.MaxCallRecvMsgSize(grpcutil.MaxMsgSize),
			grpc.MaxCallSendMsgSize(grpcutil.MaxMsgSize),
		),
	}
}

// PachDialOptions is a helper returning a slice of grpc.Dial options
// such that
// - TLS is disabled
// - Dial is synchronous: the call doesn't return until the connection has been
//                        established and it's safe to send RPCs
//
// This is primarily useful for Pachd and Worker clients
func PachDialOptions() []grpc.DialOption {
	return append(EtcdDialOptions(), grpc.WithInsecure())
}

func (c *APIClient) connect() error {
	keepaliveOpt := grpc.WithKeepaliveParams(keepalive.ClientParameters{
		Time:                20 * time.Second, // if 20s since last msg (any kind), ping
		Timeout:             20 * time.Second, // if no response to ping for 20s, reset
		PermitWithoutStream: true,             // send ping even if no active RPCs
	})
	dialOptions := append(PachDialOptions(), keepaliveOpt)
	clientConn, err := grpc.Dial(c.addr, dialOptions...)
	if err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	c.PfsAPIClient = pfs.NewAPIClient(clientConn)
	c.PpsAPIClient = pps.NewAPIClient(clientConn)
	c.ObjectAPIClient = pfs.NewObjectAPIClient(clientConn)
	c.AuthAPIClient = auth.NewAPIClient(clientConn)
	c.Enterprise = enterprise.NewAPIClient(clientConn)
	c.DeployAPIClient = deploy.NewAPIClient(clientConn)
	c.VersionAPIClient = versionpb.NewAPIClient(clientConn)
	c.clientConn = clientConn
	c.healthClient = health.NewHealthClient(clientConn)
	return nil
}

// AddMetadata adds necessary metadata (including authentication credentials)
// to the context 'ctx', preserving any metadata that is present in either the
// incoming or outgoing metadata of 'ctx'.
func (c *APIClient) AddMetadata(ctx context.Context) context.Context {
	// TODO(msteffen): There are several places in this client where it's possible
	// to set per-request metadata (specifically auth tokens): client.WithCtx(),
	// client.SetAuthToken(), etc. These should be consolidated, as this API
	// doesn't make it obvious how these settings are resolved when they conflict.
	clientData := make(map[string]string)
	if c.authenticationToken != "" {
		clientData[auth.ContextTokenKey] = c.authenticationToken
	}
	// metadata API downcases all the key names
	if c.metricsUserID != "" {
		clientData["userid"] = c.metricsUserID
		clientData["prefix"] = c.metricsPrefix
	}

	// Rescue any metadata pairs already in 'ctx' (otherwise
	// metadata.NewOutgoingContext() would drop them). Note that this is similar
	// to metadata.Join(), but distinct because it discards conflicting k/v pairs
	// instead of merging them)
	incomingMD, _ := metadata.FromIncomingContext(ctx)
	outgoingMD, _ := metadata.FromOutgoingContext(ctx)
	clientMD := metadata.New(clientData)
	finalMD := make(metadata.MD) // Collect k/v pairs
	for _, md := range []metadata.MD{incomingMD, outgoingMD, clientMD} {
		for k, v := range md {
			finalMD[k] = v
		}
	}
	return metadata.NewOutgoingContext(ctx, finalMD)
}

// Ctx is a convenience function that returns adds Pachyderm authn metadata
// to context.Background().
func (c *APIClient) Ctx() context.Context {
	if c.ctx == nil {
		return c.AddMetadata(context.Background())
	}
	return c.AddMetadata(c.ctx)
}

// WithCtx returns a new APIClient that uses ctx for requests it sends. Note
// that the new APIClient will still use the authentication token and metrics
// metadata of this client, so this is only useful for propagating other
// context-associated metadata.
func (c *APIClient) WithCtx(ctx context.Context) *APIClient {
	result := *c // copy c
	result.ctx = ctx
	return &result
}

// SetAuthToken sets the authentication token that will be used for all
// API calls for this client.
func (c *APIClient) SetAuthToken(token string) {
	c.authenticationToken = token
}
<|endoftext|>


<commit_before>package engine

import "math"

type DotsMask struct {
	mask [][]uint8
}

var DotsMaskSquare2x2 = NewDotsMask([][]uint8{
	{1, 1},
	{1, 1},
})

var DotsMaskTank = NewDotsMask([][]uint8{
	{0, 1, 0},
	{1, 1, 1},
	{1, 0, 1},
})

var DotsMaskHome = NewDotsMask([][]uint8{
	{1, 1, 0, 1, 1},
	{1, 0, 0, 0, 1},
	{1, 0, 0, 0, 1},
	{1, 0, 0, 0, 1},
	{1, 0, 0, 0, 1},
	{1, 1, 1, 1, 1},
})

func NewDotsMask(mask [][]uint8) *DotsMask {
	copyMask := make([][]uint8, len(mask))

	for i, row := range mask {
		if len(row) > math.MaxUint8 {
			copyMask[i] = make([]uint8, math.MaxUint8+1)
		} else {
			copyMask[i] = make([]uint8, len(row))
		}
		copy(copyMask[i], row)
	}

	return &DotsMask{
		mask: copyMask,
	}
}

func NewZeroDotsMask(width, height uint8) *DotsMask {
	mask := make([][]uint8, height)
	for i := range mask {
		mask[i] = make([]uint8, width)
	}
	return &DotsMask{
		mask: mask,
	}
}

func (dm *DotsMask) Copy() *DotsMask {
	copyMask := make([][]uint8, len(dm.mask))

	for i, row := range dm.mask {
		if len(row) > math.MaxUint8 {
			copyMask[i] = make([]uint8, math.MaxUint8+1)
		} else {
			copyMask[i] = make([]uint8, len(row))
		}
		copy(copyMask[i], row)
	}

	return &DotsMask{
		mask: copyMask,
	}
}

func (dm *DotsMask) Width() uint8 {
	width := 0
	for _, row := range dm.mask {
		if width < len(row) {
			width = len(row)
		}
	}
	return uint8(width)
}

func (dm *DotsMask) Height() uint8 {
	return uint8(len(dm.mask))
}

func (dm *DotsMask) TurnOver() *DotsMask {
	newMask := NewZeroDotsMask(dm.Width(), dm.Height())
	for i := range dm.mask {
		copy(newMask.mask[len(dm.mask)-1-i], dm.mask[i])
	}
	return newMask
}

func (dm *DotsMask) TurnRight() *DotsMask {
	newMask := NewZeroDotsMask(dm.Height(), dm.Width())
	for i := 0; i < len(dm.mask); i++ {
		for j := 0; j < len(dm.mask[i]); j++ {
			newMask.mask[j][len(dm.mask)-1-i] = dm.mask[i][j]
		}
	}
	return newMask
}

func (dm *DotsMask) TurnLeft() *DotsMask {
	newMask := NewZeroDotsMask(dm.Height(), dm.Width())
	for i := 0; i < len(dm.mask); i++ {
		for j := 0; j < len(dm.mask[i]); j++ {
			newMask.mask[len(newMask.mask)-1-j][i] = dm.mask[i][j]
		}
	}
	return newMask
}

func (dm *DotsMask) Location(x, y uint8) Location {
	location := make(Location, 0)
	for i := 0; i < len(dm.mask); i++ {
		for j := 0; j < len(dm.mask[i]); j++ {
			if dm.mask[i][j] > 0 {
				location = append(location, Dot{
					X: x + uint8(j),
					Y: y + uint8(i),
				})
			}
		}
	}
	return location
}
<commit_msg>Create DotsMask masks: DotsMaskCross, DotsMaskDiagonal<commit_after>package engine

import "math"

type DotsMask struct {
	mask [][]uint8
}

var DotsMaskSquare2x2 = NewDotsMask([][]uint8{
	{1, 1},
	{1, 1},
})

var DotsMaskTank = NewDotsMask([][]uint8{
	{0, 1, 0},
	{1, 1, 1},
	{1, 0, 1},
})

var DotsMaskHome = NewDotsMask([][]uint8{
	{1, 1, 0, 1, 1},
	{1, 0, 0, 0, 1},
	{1, 0, 0, 0, 1},
	{1, 0, 0, 0, 1},
	{1, 0, 0, 0, 1},
	{1, 1, 1, 1, 1},
})

var DotsMaskCross = NewDotsMask([][]uint8{
	{0, 0, 0, 1, 0, 0, 0},
	{0, 0, 0, 1, 0, 0, 0},
	{0, 0, 0, 1, 0, 0, 0},
	{1, 1, 1, 1, 1, 1, 1},
	{0, 0, 0, 1, 0, 0, 0},
	{0, 0, 0, 1, 0, 0, 0},
	{0, 0, 0, 1, 0, 0, 0},
})

var DotsMaskDiagonal = NewDotsMask([][]uint8{
	{0, 0, 0, 0, 0, 0, 1},
	{0, 0, 0, 0, 0, 1, 0},
	{0, 0, 0, 0, 1, 0, 0},
	{0, 0, 0, 1, 0, 0, 0},
	{0, 0, 1, 0, 0, 0, 0},
	{0, 1, 0, 0, 0, 0, 0},
	{1, 0, 0, 0, 0, 0, 0},
})

func NewDotsMask(mask [][]uint8) *DotsMask {
	copyMask := make([][]uint8, len(mask))

	for i, row := range mask {
		if len(row) > math.MaxUint8 {
			copyMask[i] = make([]uint8, math.MaxUint8+1)
		} else {
			copyMask[i] = make([]uint8, len(row))
		}
		copy(copyMask[i], row)
	}

	return &DotsMask{
		mask: copyMask,
	}
}

func NewZeroDotsMask(width, height uint8) *DotsMask {
	mask := make([][]uint8, height)
	for i := range mask {
		mask[i] = make([]uint8, width)
	}
	return &DotsMask{
		mask: mask,
	}
}

func (dm *DotsMask) Copy() *DotsMask {
	copyMask := make([][]uint8, len(dm.mask))

	for i, row := range dm.mask {
		if len(row) > math.MaxUint8 {
			copyMask[i] = make([]uint8, math.MaxUint8+1)
		} else {
			copyMask[i] = make([]uint8, len(row))
		}
		copy(copyMask[i], row)
	}

	return &DotsMask{
		mask: copyMask,
	}
}

func (dm *DotsMask) Width() uint8 {
	width := 0
	for _, row := range dm.mask {
		if width < len(row) {
			width = len(row)
		}
	}
	return uint8(width)
}

func (dm *DotsMask) Height() uint8 {
	return uint8(len(dm.mask))
}

func (dm *DotsMask) TurnOver() *DotsMask {
	newMask := NewZeroDotsMask(dm.Width(), dm.Height())
	for i := range dm.mask {
		copy(newMask.mask[len(dm.mask)-1-i], dm.mask[i])
	}
	return newMask
}

func (dm *DotsMask) TurnRight() *DotsMask {
	newMask := NewZeroDotsMask(dm.Height(), dm.Width())
	for i := 0; i < len(dm.mask); i++ {
		for j := 0; j < len(dm.mask[i]); j++ {
			newMask.mask[j][len(dm.mask)-1-i] = dm.mask[i][j]
		}
	}
	return newMask
}

func (dm *DotsMask) TurnLeft() *DotsMask {
	newMask := NewZeroDotsMask(dm.Height(), dm.Width())
	for i := 0; i < len(dm.mask); i++ {
		for j := 0; j < len(dm.mask[i]); j++ {
			newMask.mask[len(newMask.mask)-1-j][i] = dm.mask[i][j]
		}
	}
	return newMask
}

func (dm *DotsMask) Location(x, y uint8) Location {
	location := make(Location, 0)
	for i := 0; i < len(dm.mask); i++ {
		for j := 0; j < len(dm.mask[i]); j++ {
			if dm.mask[i][j] > 0 {
				location = append(location, Dot{
					X: x + uint8(j),
					Y: y + uint8(i),
				})
			}
		}
	}
	return location
}
<|endoftext|>


<commit_before>package virt

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"path/filepath"
	"strings"
	"time"

	"github.com/projecteru2/core/cluster"
	"github.com/projecteru2/core/engine"
	enginetypes "github.com/projecteru2/core/engine/types"
	"github.com/projecteru2/core/log"
	coresource "github.com/projecteru2/core/source"
	"github.com/projecteru2/core/types"
	coretypes "github.com/projecteru2/core/types"
	virtapi "github.com/projecteru2/libyavirt/client"
	virttypes "github.com/projecteru2/libyavirt/types"
)

const (
	// HTTPPrefixKey indicate http yavirtd
	HTTPPrefixKey = "virt://"
	// GRPCPrefixKey indicates grpc yavirtd
	GRPCPrefixKey = "virt-grpc://"
	// DmiUUIDKey indicates the key within deploy info.
	DmiUUIDKey = "DMIUUID"
	// ImageUserKey indicates the image's owner
	ImageUserKey = "ImageUser"
)

// Virt implements the core engine.API interface.
type Virt struct {
	client virtapi.Client
	config coretypes.Config
}

// MakeClient makes a virt. client which wraps yavirt API client.
func MakeClient(ctx context.Context, config coretypes.Config, nodename, endpoint, ca, cert, key string) (engine.API, error) {
	var uri string
	switch {
	case strings.HasPrefix(endpoint, HTTPPrefixKey):
		uri = fmt.Sprintf("http://%s/%s", strings.TrimPrefix(endpoint, HTTPPrefixKey), config.Virt.APIVersion)
	case strings.HasPrefix(endpoint, GRPCPrefixKey):
		uri = "grpc://" + strings.TrimPrefix(endpoint, GRPCPrefixKey)
	default:
		return nil, fmt.Errorf("invalid endpoint: %s", endpoint)
	}

	cli, err := virtapi.New(uri)
	if err != nil {
		return nil, err
	}
	return &Virt{cli, config}, nil
}

// Info shows a connected node's information.
func (v *Virt) Info(ctx context.Context) (*enginetypes.Info, error) {
	resp, err := v.client.Info(ctx)
	if err != nil {
		return nil, err
	}

	return &enginetypes.Info{
		ID:           resp.ID,
		NCPU:         resp.CPU,
		MemTotal:     resp.Mem,
		StorageTotal: resp.Storage,
	}, nil
}

// Execute executes a command in vm
func (v *Virt) Execute(ctx context.Context, target string, config *enginetypes.ExecConfig) (execID string, stdout io.ReadCloser, stderr io.ReadCloser, inputStream io.WriteCloser, err error) {
	if config.Tty {
		flags := virttypes.AttachGuestFlags{Safe: true, Force: true}
		stream, err := v.client.AttachGuest(ctx, target, config.Cmd, flags)
		if err != nil {
			return "", nil, nil, nil, err
		}
		return target, ioutil.NopCloser(stream), nil, stream, nil

	}

	msg, err := v.client.ExecuteGuest(ctx, target, config.Cmd)
	return target, ioutil.NopCloser(bytes.NewReader(msg.Data)), nil, nil, err

}

// ExecExitCode gets return code of a specific execution.
func (v *Virt) ExecExitCode(ctx context.Context, execID string) (code int, err error) {
	return 0, nil
}

// ExecResize resize exec tty
func (v *Virt) ExecResize(ctx context.Context, execID string, height, width uint) (err error) {
	return v.client.ResizeConsoleWindow(ctx, execID, height, width)
}

// NetworkConnect connects to a network.
func (v *Virt) NetworkConnect(ctx context.Context, network, target, ipv4, ipv6 string) (cidrs []string, err error) {
	req := virttypes.ConnectNetworkReq{
		Network: network,
		IPv4:    ipv4,
	}
	req.ID = target

	var cidr string
	if cidr, err = v.client.ConnectNetwork(ctx, req); err != nil {
		return
	}

	cidrs = append(cidrs, cidr)

	return
}

// NetworkDisconnect disconnects from one network.
func (v *Virt) NetworkDisconnect(ctx context.Context, network, target string, force bool) (err error) {
	var req virttypes.DisconnectNetworkReq
	req.Network = network
	req.ID = target

	_, err = v.client.DisconnectNetwork(ctx, req)

	return
}

// NetworkList lists all of networks.
func (v *Virt) NetworkList(ctx context.Context, drivers []string) (nets []*enginetypes.Network, err error) {
	log.Warnf(ctx, "NetworkList does not implement")
	return
}

// BuildRefs builds references, it's not necessary for virt. presently.
func (v *Virt) BuildRefs(ctx context.Context, name string, tags []string) (refs []string) {
	log.Warnf(ctx, "BuildRefs does not implement")
	return
}

// BuildContent builds content, the use of it is similar to BuildRefs.
func (v *Virt) BuildContent(ctx context.Context, scm coresource.Source, opts *enginetypes.BuildContentOptions) (string, io.Reader, error) {
	return "", nil, fmt.Errorf("BuildContent does not implement")
}

// VirtualizationCreate creates a guest.
func (v *Virt) VirtualizationCreate(ctx context.Context, opts *enginetypes.VirtualizationCreateOptions) (guest *enginetypes.VirtualizationCreated, err error) {
	vols, err := v.parseVolumes(opts.Volumes)
	if err != nil {
		return nil, err
	}

	req := virttypes.CreateGuestReq{
		CPU:        int(opts.Quota),
		Mem:        opts.Memory,
		ImageName:  opts.Image,
		ImageUser:  opts.Labels[ImageUserKey],
		Volumes:    vols,
		Labels:     opts.Labels,
		AncestorID: opts.AncestorWorkloadID,
		DmiUUID:    opts.Labels[DmiUUIDKey],
	}

	var resp virttypes.Guest
	if resp, err = v.client.CreateGuest(ctx, req); err != nil {
		return nil, err
	}

	return &enginetypes.VirtualizationCreated{ID: resp.ID, Name: opts.Name}, nil
}

// VirtualizationResourceRemap .
func (v *Virt) VirtualizationResourceRemap(ctx context.Context, opts *enginetypes.VirtualizationRemapOptions) (ch <-chan enginetypes.VirtualizationRemapMessage, err error) {
	err = types.ErrEngineNotImplemented
	return
}

// VirtualizationCopyTo copies one.
func (v *Virt) VirtualizationCopyTo(ctx context.Context, ID, dest string, content io.Reader, AllowOverwriteDirWithFile, CopyUIDGID bool) error {
	return v.client.CopyToGuest(ctx, ID, dest, content, AllowOverwriteDirWithFile, CopyUIDGID)
}

// VirtualizationStart boots a guest.
func (v *Virt) VirtualizationStart(ctx context.Context, ID string) (err error) {
	_, err = v.client.StartGuest(ctx, ID)
	return
}

// VirtualizationStop stops it.
func (v *Virt) VirtualizationStop(ctx context.Context, ID string, gracefulTimeout time.Duration) (err error) {
	_, err = v.client.StopGuest(ctx, ID, gracefulTimeout == 0)
	return
}

// VirtualizationRemove removes a guest.
func (v *Virt) VirtualizationRemove(ctx context.Context, ID string, volumes, force bool) (err error) {
	_, err = v.client.DestroyGuest(ctx, ID, force)
	return
}

// VirtualizationInspect gets a guest.
func (v *Virt) VirtualizationInspect(ctx context.Context, ID string) (*enginetypes.VirtualizationInfo, error) {
	guest, err := v.client.GetGuest(ctx, ID)
	if err != nil {
		return nil, err
	}

	content, err := json.Marshal(coretypes.LabelMeta{Publish: []string{"PORT"}})
	if err != nil {
		return nil, err
	}

	return &enginetypes.VirtualizationInfo{
		ID:       guest.ID,
		Image:    guest.ImageName,
		Running:  guest.Status == "running",
		Networks: guest.Networks,
		Labels:   map[string]string{cluster.LabelMeta: string(content), cluster.ERUMark: "1"},
	}, nil
}

// VirtualizationLogs streams a specific guest's log.
func (v *Virt) VirtualizationLogs(ctx context.Context, opts *enginetypes.VirtualizationLogStreamOptions) (stdout io.ReadCloser, stderr io.ReadCloser, err error) {
	return nil, nil, fmt.Errorf("VirtualizationLogs does not implement")
}

// VirtualizationAttach attaches something to a guest.
func (v *Virt) VirtualizationAttach(ctx context.Context, ID string, stream, stdin bool) (io.ReadCloser, io.ReadCloser, io.WriteCloser, error) {
	return nil, nil, nil, fmt.Errorf("VirtualizationAttach does not implement")
}

// VirtualizationResize resized window size
func (v *Virt) VirtualizationResize(ctx context.Context, ID string, height, width uint) error {
	return fmt.Errorf("VirtualizationResize not implemented")
}

// VirtualizationWait is waiting for a shut-off
func (v *Virt) VirtualizationWait(ctx context.Context, ID, state string) (*enginetypes.VirtualizationWaitResult, error) {
	return nil, fmt.Errorf("VirtualizationWait does not implement")
}

// VirtualizationUpdateResource updates resource.
func (v *Virt) VirtualizationUpdateResource(ctx context.Context, ID string, opts *enginetypes.VirtualizationResource) error {
	vols, err := v.parseVolumes(opts.Volumes)
	if err != nil {
		return err
	}

	args := virttypes.ResizeGuestReq{
		CPU:     int(opts.Quota),
		Mem:     opts.Memory,
		Volumes: vols,
	}
	args.ID = ID

	_, err = v.client.ResizeGuest(ctx, args)
	return err
}

// VirtualizationCopyFrom copies file content from the container.
func (v *Virt) VirtualizationCopyFrom(ctx context.Context, ID, path string) (io.ReadCloser, string, error) {
	rd, err := v.client.Cat(ctx, ID, path)
	if err != nil {
		return nil, "", err
	}
	return ioutil.NopCloser(rd), filepath.Base(path), nil
}

// VirtualizationExecute executes commands in running virtual unit
func (v *Virt) VirtualizationExecute(ctx context.Context, ID string, commands, env []string, workdir string) (io.WriteCloser, io.ReadCloser, error) {
	return nil, nil, fmt.Errorf("VirtualizationExecute not implemented")
}

// ResourceValidate validate resource usage
func (v *Virt) ResourceValidate(ctx context.Context, cpu float64, cpumap map[string]int64, memory, storage int64) error {
	// TODO list all workloads, calculate resource
	return nil
}
<commit_msg>add resize (#478)<commit_after>package virt

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"path/filepath"
	"strings"
	"time"

	"github.com/projecteru2/core/cluster"
	"github.com/projecteru2/core/engine"
	enginetypes "github.com/projecteru2/core/engine/types"
	"github.com/projecteru2/core/log"
	coresource "github.com/projecteru2/core/source"
	"github.com/projecteru2/core/types"
	coretypes "github.com/projecteru2/core/types"
	virtapi "github.com/projecteru2/libyavirt/client"
	virttypes "github.com/projecteru2/libyavirt/types"
)

const (
	// HTTPPrefixKey indicate http yavirtd
	HTTPPrefixKey = "virt://"
	// GRPCPrefixKey indicates grpc yavirtd
	GRPCPrefixKey = "virt-grpc://"
	// DmiUUIDKey indicates the key within deploy info.
	DmiUUIDKey = "DMIUUID"
	// ImageUserKey indicates the image's owner
	ImageUserKey = "ImageUser"
)

// Virt implements the core engine.API interface.
type Virt struct {
	client virtapi.Client
	config coretypes.Config
}

// MakeClient makes a virt. client which wraps yavirt API client.
func MakeClient(ctx context.Context, config coretypes.Config, nodename, endpoint, ca, cert, key string) (engine.API, error) {
	var uri string
	switch {
	case strings.HasPrefix(endpoint, HTTPPrefixKey):
		uri = fmt.Sprintf("http://%s/%s", strings.TrimPrefix(endpoint, HTTPPrefixKey), config.Virt.APIVersion)
	case strings.HasPrefix(endpoint, GRPCPrefixKey):
		uri = "grpc://" + strings.TrimPrefix(endpoint, GRPCPrefixKey)
	default:
		return nil, fmt.Errorf("invalid endpoint: %s", endpoint)
	}

	cli, err := virtapi.New(uri)
	if err != nil {
		return nil, err
	}
	return &Virt{cli, config}, nil
}

// Info shows a connected node's information.
func (v *Virt) Info(ctx context.Context) (*enginetypes.Info, error) {
	resp, err := v.client.Info(ctx)
	if err != nil {
		return nil, err
	}

	return &enginetypes.Info{
		ID:           resp.ID,
		NCPU:         resp.CPU,
		MemTotal:     resp.Mem,
		StorageTotal: resp.Storage,
	}, nil
}

// Execute executes a command in vm
func (v *Virt) Execute(ctx context.Context, target string, config *enginetypes.ExecConfig) (execID string, stdout io.ReadCloser, stderr io.ReadCloser, inputStream io.WriteCloser, err error) {
	if config.Tty {
		flags := virttypes.AttachGuestFlags{Safe: true, Force: true}
		stream, err := v.client.AttachGuest(ctx, target, config.Cmd, flags)
		if err != nil {
			return "", nil, nil, nil, err
		}
		return target, ioutil.NopCloser(stream), nil, stream, nil

	}

	msg, err := v.client.ExecuteGuest(ctx, target, config.Cmd)
	return target, ioutil.NopCloser(bytes.NewReader(msg.Data)), nil, nil, err

}

// ExecExitCode gets return code of a specific execution.
func (v *Virt) ExecExitCode(ctx context.Context, execID string) (code int, err error) {
	return 0, nil
}

// ExecResize resize exec tty
func (v *Virt) ExecResize(ctx context.Context, execID string, height, width uint) (err error) {
	return v.client.ResizeConsoleWindow(ctx, execID, height, width)
}

// NetworkConnect connects to a network.
func (v *Virt) NetworkConnect(ctx context.Context, network, target, ipv4, ipv6 string) (cidrs []string, err error) {
	req := virttypes.ConnectNetworkReq{
		Network: network,
		IPv4:    ipv4,
	}
	req.ID = target

	var cidr string
	if cidr, err = v.client.ConnectNetwork(ctx, req); err != nil {
		return
	}

	cidrs = append(cidrs, cidr)

	return
}

// NetworkDisconnect disconnects from one network.
func (v *Virt) NetworkDisconnect(ctx context.Context, network, target string, force bool) (err error) {
	var req virttypes.DisconnectNetworkReq
	req.Network = network
	req.ID = target

	_, err = v.client.DisconnectNetwork(ctx, req)

	return
}

// NetworkList lists all of networks.
func (v *Virt) NetworkList(ctx context.Context, drivers []string) (nets []*enginetypes.Network, err error) {
	log.Warnf(ctx, "NetworkList does not implement")
	return
}

// BuildRefs builds references, it's not necessary for virt. presently.
func (v *Virt) BuildRefs(ctx context.Context, name string, tags []string) (refs []string) {
	log.Warnf(ctx, "BuildRefs does not implement")
	return
}

// BuildContent builds content, the use of it is similar to BuildRefs.
func (v *Virt) BuildContent(ctx context.Context, scm coresource.Source, opts *enginetypes.BuildContentOptions) (string, io.Reader, error) {
	return "", nil, fmt.Errorf("BuildContent does not implement")
}

// VirtualizationCreate creates a guest.
func (v *Virt) VirtualizationCreate(ctx context.Context, opts *enginetypes.VirtualizationCreateOptions) (guest *enginetypes.VirtualizationCreated, err error) {
	vols, err := v.parseVolumes(opts.Volumes)
	if err != nil {
		return nil, err
	}

	req := virttypes.CreateGuestReq{
		CPU:        int(opts.Quota),
		Mem:        opts.Memory,
		ImageName:  opts.Image,
		ImageUser:  opts.Labels[ImageUserKey],
		Volumes:    vols,
		Labels:     opts.Labels,
		AncestorID: opts.AncestorWorkloadID,
		DmiUUID:    opts.Labels[DmiUUIDKey],
	}

	var resp virttypes.Guest
	if resp, err = v.client.CreateGuest(ctx, req); err != nil {
		return nil, err
	}

	return &enginetypes.VirtualizationCreated{ID: resp.ID, Name: opts.Name}, nil
}

// VirtualizationResourceRemap .
func (v *Virt) VirtualizationResourceRemap(ctx context.Context, opts *enginetypes.VirtualizationRemapOptions) (ch <-chan enginetypes.VirtualizationRemapMessage, err error) {
	err = types.ErrEngineNotImplemented
	return
}

// VirtualizationCopyTo copies one.
func (v *Virt) VirtualizationCopyTo(ctx context.Context, ID, dest string, content io.Reader, AllowOverwriteDirWithFile, CopyUIDGID bool) error {
	return v.client.CopyToGuest(ctx, ID, dest, content, AllowOverwriteDirWithFile, CopyUIDGID)
}

// VirtualizationStart boots a guest.
func (v *Virt) VirtualizationStart(ctx context.Context, ID string) (err error) {
	_, err = v.client.StartGuest(ctx, ID)
	return
}

// VirtualizationStop stops it.
func (v *Virt) VirtualizationStop(ctx context.Context, ID string, gracefulTimeout time.Duration) (err error) {
	_, err = v.client.StopGuest(ctx, ID, gracefulTimeout == 0)
	return
}

// VirtualizationRemove removes a guest.
func (v *Virt) VirtualizationRemove(ctx context.Context, ID string, volumes, force bool) (err error) {
	_, err = v.client.DestroyGuest(ctx, ID, force)
	return
}

// VirtualizationInspect gets a guest.
func (v *Virt) VirtualizationInspect(ctx context.Context, ID string) (*enginetypes.VirtualizationInfo, error) {
	guest, err := v.client.GetGuest(ctx, ID)
	if err != nil {
		return nil, err
	}

	content, err := json.Marshal(coretypes.LabelMeta{Publish: []string{"PORT"}})
	if err != nil {
		return nil, err
	}

	return &enginetypes.VirtualizationInfo{
		ID:       guest.ID,
		Image:    guest.ImageName,
		Running:  guest.Status == "running",
		Networks: guest.Networks,
		Labels:   map[string]string{cluster.LabelMeta: string(content), cluster.ERUMark: "1"},
	}, nil
}

// VirtualizationLogs streams a specific guest's log.
func (v *Virt) VirtualizationLogs(ctx context.Context, opts *enginetypes.VirtualizationLogStreamOptions) (stdout io.ReadCloser, stderr io.ReadCloser, err error) {
	return nil, nil, fmt.Errorf("VirtualizationLogs does not implement")
}

// VirtualizationAttach attaches something to a guest.
func (v *Virt) VirtualizationAttach(ctx context.Context, ID string, stream, stdin bool) (io.ReadCloser, io.ReadCloser, io.WriteCloser, error) {
	return nil, nil, nil, fmt.Errorf("VirtualizationAttach does not implement")
}

// VirtualizationResize resized window size
func (v *Virt) VirtualizationResize(ctx context.Context, ID string, height, width uint) error {
	return v.client.ResizeConsoleWindow(ctx, ID, height, width)
}

// VirtualizationWait is waiting for a shut-off
func (v *Virt) VirtualizationWait(ctx context.Context, ID, state string) (*enginetypes.VirtualizationWaitResult, error) {
	return nil, fmt.Errorf("VirtualizationWait does not implement")
}

// VirtualizationUpdateResource updates resource.
func (v *Virt) VirtualizationUpdateResource(ctx context.Context, ID string, opts *enginetypes.VirtualizationResource) error {
	vols, err := v.parseVolumes(opts.Volumes)
	if err != nil {
		return err
	}

	args := virttypes.ResizeGuestReq{
		CPU:     int(opts.Quota),
		Mem:     opts.Memory,
		Volumes: vols,
	}
	args.ID = ID

	_, err = v.client.ResizeGuest(ctx, args)
	return err
}

// VirtualizationCopyFrom copies file content from the container.
func (v *Virt) VirtualizationCopyFrom(ctx context.Context, ID, path string) (io.ReadCloser, string, error) {
	rd, err := v.client.Cat(ctx, ID, path)
	if err != nil {
		return nil, "", err
	}
	return ioutil.NopCloser(rd), filepath.Base(path), nil
}

// VirtualizationExecute executes commands in running virtual unit
func (v *Virt) VirtualizationExecute(ctx context.Context, ID string, commands, env []string, workdir string) (io.WriteCloser, io.ReadCloser, error) {
	return nil, nil, fmt.Errorf("VirtualizationExecute not implemented")
}

// ResourceValidate validate resource usage
func (v *Virt) ResourceValidate(ctx context.Context, cpu float64, cpumap map[string]int64, memory, storage int64) error {
	// TODO list all workloads, calculate resource
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package localfs

import (
	"log"
	"os"
	"path"
	"syscall"

	"github.com/hugelgupf/p9/p9"
	"github.com/hugelgupf/p9/unimplfs"
)

// Local is a p9.Attacher.
type Local struct {
	p9.DefaultWalkGetAttr
	unimplfs.NoopFile

	path string
	file *os.File
}

// Attach implements p9.Attacher.Attach.
func (l *Local) Attach() (p9.File, error) {
	return &Local{path: "/"}, nil
}

var (
	_ p9.File     = &Local{}
	_ p9.Attacher = &Local{}
)

// info constructs a QID for this file.
func (l *Local) info() (p9.QID, os.FileInfo, error) {
	var (
		qid p9.QID
		fi  os.FileInfo
		err error
	)

	// Stat the file.
	if l.file != nil {
		fi, err = l.file.Stat()
	} else {
		fi, err = os.Lstat(l.path)
	}
	if err != nil {
		log.Printf("error stating %#v: %v", l, err)
		return qid, nil, err
	}

	// Construct the QID type.
	qid.Type = p9.ModeFromOS(fi.Mode()).QIDType()

	// Save the path from the Ino.
	qid.Path = fi.Sys().(*syscall.Stat_t).Ino
	return qid, fi, nil
}

// Walk implements p9.File.Walk.
func (l *Local) Walk(names []string) ([]p9.QID, p9.File, error) {
	var qids []p9.QID
	last := &Local{path: l.path}
	for _, name := range names {
		c := &Local{path: path.Join(last.path, name)}
		qid, _, err := c.info()
		if err != nil {
			return nil, nil, err
		}
		qids = append(qids, qid)
		last = c
	}
	return qids, last, nil
}

// FSync implements p9.File.FSync.
func (l *Local) FSync() error {
	return l.file.Sync()
}

// GetAttr implements p9.File.GetAttr.
//
// Not fully implemented.
func (l *Local) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) {
	qid, fi, err := l.info()
	if err != nil {
		return qid, p9.AttrMask{}, p9.Attr{}, err
	}

	stat := fi.Sys().(*syscall.Stat_t)
	attr := p9.Attr{
		Mode:             p9.FileMode(stat.Mode),
		UID:              p9.UID(stat.Uid),
		GID:              p9.GID(stat.Gid),
		NLink:            stat.Nlink,
		RDev:             stat.Rdev,
		Size:             uint64(stat.Size),
		BlockSize:        uint64(stat.Blksize),
		Blocks:           uint64(stat.Blocks),
		ATimeSeconds:     uint64(stat.Atim.Sec),
		ATimeNanoSeconds: uint64(stat.Atim.Nsec),
		MTimeSeconds:     uint64(stat.Mtim.Sec),
		MTimeNanoSeconds: uint64(stat.Mtim.Nsec),
		CTimeSeconds:     uint64(stat.Ctim.Sec),
		CTimeNanoSeconds: uint64(stat.Ctim.Nsec),
	}
	valid := p9.AttrMask{
		Mode:   true,
		UID:    true,
		GID:    true,
		NLink:  true,
		RDev:   true,
		Size:   true,
		Blocks: true,
		ATime:  true,
		MTime:  true,
		CTime:  true,
	}

	return qid, valid, attr, nil
}

// Close implements p9.File.Close.
func (l *Local) Close() error {
	if l.file != nil {
		return l.file.Close()
	}
	return nil
}

// Open implements p9.File.Open.
func (l *Local) Open(mode p9.OpenFlags) (p9.QID, uint32, error) {
	qid, _, err := l.info()
	if err != nil {
		return qid, 0, err
	}

	// Do the actual open.
	f, err := os.OpenFile(l.path, int(mode), 0)
	if err != nil {
		return qid, 0, err
	}
	l.file = f

	return qid, 4096, nil
}

// Read implements p9.File.Read.
func (l *Local) ReadAt(p []byte, offset uint64) (int, error) {
	return l.file.ReadAt(p, int64(offset))
}

// Write implements p9.File.Write.
func (l *Local) WriteAt(p []byte, offset uint64) (int, error) {
	return l.file.WriteAt(p, int64(offset))
}

// Create implements p9.File.Create.
func (l *Local) Create(name string, mode p9.OpenFlags, permissions p9.FileMode, _ p9.UID, _ p9.GID) (p9.File, p9.QID, uint32, error) {
	f, err := os.OpenFile(l.path, int(mode)|syscall.O_CREAT|syscall.O_EXCL, os.FileMode(permissions))
	if err != nil {
		return nil, p9.QID{}, 0, err
	}

	l2 := &Local{path: path.Join(l.path, name), file: f}
	qid, _, err := l2.info()
	if err != nil {
		l2.Close()
		return nil, p9.QID{}, 0, err
	}

	return l2, qid, 4096, nil
}

// Mkdir implements p9.File.Mkdir.
//
// Not properly implemented.
func (l *Local) Mkdir(name string, permissions p9.FileMode, _ p9.UID, _ p9.GID) (p9.QID, error) {
	if err := os.Mkdir(path.Join(l.path, name), os.FileMode(permissions)); err != nil {
		return p9.QID{}, err
	}

	// Blank QID.
	return p9.QID{}, nil
}

// Symlink implements p9.File.Symlink.
//
// Not properly implemented.
func (l *Local) Symlink(oldname string, newname string, _ p9.UID, _ p9.GID) (p9.QID, error) {
	if err := os.Symlink(oldname, path.Join(l.path, newname)); err != nil {
		return p9.QID{}, err
	}

	// Blank QID.
	return p9.QID{}, nil
}

// Link implements p9.File.Link.
//
// Not properly implemented.
func (l *Local) Link(target p9.File, newname string) error {
	return os.Link(target.(*Local).path, path.Join(l.path, newname))
}

// Readdir implements p9.File.Readdir.
func (l *Local) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) {
	// We only do *all* dirents in single shot.
	const maxDirentBuffer = 1024 * 1024
	buf := make([]byte, maxDirentBuffer)
	n, err := syscall.ReadDirent(int(l.file.Fd()), buf)
	if err != nil {
		// Return zero entries.
		return nil, nil
	}

	// Parse the entries; note that we read up to offset+count here.
	_, newCount, newNames := syscall.ParseDirent(buf[:n], int(offset)+int(count), nil)
	var dirents []p9.Dirent
	for i := int(offset); i >= 0 && i < newCount; i++ {
		entry := Local{path: path.Join(l.path, newNames[i])}
		qid, _, err := entry.info()
		if err != nil {
			continue
		}
		dirents = append(dirents, p9.Dirent{
			QID:    qid,
			Type:   qid.Type,
			Name:   newNames[i],
			Offset: uint64(i + 1),
		})
	}

	return dirents, nil
}

// Readlink implements p9.File.Readlink.
//
// Not properly implemented.
func (l *Local) Readlink() (string, error) {
	return os.Readlink(l.path)
}

// Flush implements p9.File.Flush.
func (l *Local) Flush() error {
	return nil
}

// Renamed implements p9.File.Renamed.
func (l *Local) Renamed(parent p9.File, newName string) {
	l.path = path.Join(parent.(*Local).path, newName)
}

// Allocate implements p9.File.Allocate.
func (l *Local) Allocate(mode p9.AllocateMode, offset, length uint64) error {
	return syscall.Fallocate(int(l.file.Fd()), mode.ToLinux(), int64(offset), int64(length))
}
<commit_msg>localfs: allow walks to self<commit_after>// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package localfs

import (
	"log"
	"os"
	"path"
	"syscall"

	"github.com/hugelgupf/p9/p9"
	"github.com/hugelgupf/p9/unimplfs"
)

// Local is a p9.Attacher.
type Local struct {
	p9.DefaultWalkGetAttr
	unimplfs.NoopFile

	path string
	file *os.File
}

// Attach implements p9.Attacher.Attach.
func (l *Local) Attach() (p9.File, error) {
	return &Local{path: "/"}, nil
}

var (
	_ p9.File     = &Local{}
	_ p9.Attacher = &Local{}
)

// info constructs a QID for this file.
func (l *Local) info() (p9.QID, os.FileInfo, error) {
	var (
		qid p9.QID
		fi  os.FileInfo
		err error
	)

	// Stat the file.
	if l.file != nil {
		fi, err = l.file.Stat()
	} else {
		fi, err = os.Lstat(l.path)
	}
	if err != nil {
		log.Printf("error stating %#v: %v", l, err)
		return qid, nil, err
	}

	// Construct the QID type.
	qid.Type = p9.ModeFromOS(fi.Mode()).QIDType()

	// Save the path from the Ino.
	qid.Path = fi.Sys().(*syscall.Stat_t).Ino
	return qid, fi, nil
}

// Walk implements p9.File.Walk.
func (l *Local) Walk(names []string) ([]p9.QID, p9.File, error) {
	var qids []p9.QID
	last := &Local{path: l.path}

	// A walk with no names is a copy of self.
	if len(names) == 0 {
		qid, _, err := l.info()
		if err != nil {
			return nil, nil, err
		}
		return []p9.QID{qid}, last, nil
	}

	for _, name := range names {
		c := &Local{path: path.Join(last.path, name)}
		qid, _, err := c.info()
		if err != nil {
			return nil, nil, err
		}
		qids = append(qids, qid)
		last = c
	}
	return qids, last, nil
}

// FSync implements p9.File.FSync.
func (l *Local) FSync() error {
	return l.file.Sync()
}

// GetAttr implements p9.File.GetAttr.
//
// Not fully implemented.
func (l *Local) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) {
	qid, fi, err := l.info()
	if err != nil {
		return qid, p9.AttrMask{}, p9.Attr{}, err
	}

	stat := fi.Sys().(*syscall.Stat_t)
	attr := p9.Attr{
		Mode:             p9.FileMode(stat.Mode),
		UID:              p9.UID(stat.Uid),
		GID:              p9.GID(stat.Gid),
		NLink:            stat.Nlink,
		RDev:             stat.Rdev,
		Size:             uint64(stat.Size),
		BlockSize:        uint64(stat.Blksize),
		Blocks:           uint64(stat.Blocks),
		ATimeSeconds:     uint64(stat.Atim.Sec),
		ATimeNanoSeconds: uint64(stat.Atim.Nsec),
		MTimeSeconds:     uint64(stat.Mtim.Sec),
		MTimeNanoSeconds: uint64(stat.Mtim.Nsec),
		CTimeSeconds:     uint64(stat.Ctim.Sec),
		CTimeNanoSeconds: uint64(stat.Ctim.Nsec),
	}
	valid := p9.AttrMask{
		Mode:   true,
		UID:    true,
		GID:    true,
		NLink:  true,
		RDev:   true,
		Size:   true,
		Blocks: true,
		ATime:  true,
		MTime:  true,
		CTime:  true,
	}

	return qid, valid, attr, nil
}

// Close implements p9.File.Close.
func (l *Local) Close() error {
	if l.file != nil {
		return l.file.Close()
	}
	return nil
}

// Open implements p9.File.Open.
func (l *Local) Open(mode p9.OpenFlags) (p9.QID, uint32, error) {
	qid, _, err := l.info()
	if err != nil {
		return qid, 0, err
	}

	// Do the actual open.
	f, err := os.OpenFile(l.path, int(mode), 0)
	if err != nil {
		return qid, 0, err
	}
	l.file = f

	return qid, 4096, nil
}

// Read implements p9.File.Read.
func (l *Local) ReadAt(p []byte, offset uint64) (int, error) {
	return l.file.ReadAt(p, int64(offset))
}

// Write implements p9.File.Write.
func (l *Local) WriteAt(p []byte, offset uint64) (int, error) {
	return l.file.WriteAt(p, int64(offset))
}

// Create implements p9.File.Create.
func (l *Local) Create(name string, mode p9.OpenFlags, permissions p9.FileMode, _ p9.UID, _ p9.GID) (p9.File, p9.QID, uint32, error) {
	f, err := os.OpenFile(l.path, int(mode)|syscall.O_CREAT|syscall.O_EXCL, os.FileMode(permissions))
	if err != nil {
		return nil, p9.QID{}, 0, err
	}

	l2 := &Local{path: path.Join(l.path, name), file: f}
	qid, _, err := l2.info()
	if err != nil {
		l2.Close()
		return nil, p9.QID{}, 0, err
	}

	return l2, qid, 4096, nil
}

// Mkdir implements p9.File.Mkdir.
//
// Not properly implemented.
func (l *Local) Mkdir(name string, permissions p9.FileMode, _ p9.UID, _ p9.GID) (p9.QID, error) {
	if err := os.Mkdir(path.Join(l.path, name), os.FileMode(permissions)); err != nil {
		return p9.QID{}, err
	}

	// Blank QID.
	return p9.QID{}, nil
}

// Symlink implements p9.File.Symlink.
//
// Not properly implemented.
func (l *Local) Symlink(oldname string, newname string, _ p9.UID, _ p9.GID) (p9.QID, error) {
	if err := os.Symlink(oldname, path.Join(l.path, newname)); err != nil {
		return p9.QID{}, err
	}

	// Blank QID.
	return p9.QID{}, nil
}

// Link implements p9.File.Link.
//
// Not properly implemented.
func (l *Local) Link(target p9.File, newname string) error {
	return os.Link(target.(*Local).path, path.Join(l.path, newname))
}

// Readdir implements p9.File.Readdir.
func (l *Local) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) {
	// We only do *all* dirents in single shot.
	const maxDirentBuffer = 1024 * 1024
	buf := make([]byte, maxDirentBuffer)
	n, err := syscall.ReadDirent(int(l.file.Fd()), buf)
	if err != nil {
		// Return zero entries.
		return nil, nil
	}

	// Parse the entries; note that we read up to offset+count here.
	_, newCount, newNames := syscall.ParseDirent(buf[:n], int(offset)+int(count), nil)
	var dirents []p9.Dirent
	for i := int(offset); i >= 0 && i < newCount; i++ {
		entry := Local{path: path.Join(l.path, newNames[i])}
		qid, _, err := entry.info()
		if err != nil {
			continue
		}
		dirents = append(dirents, p9.Dirent{
			QID:    qid,
			Type:   qid.Type,
			Name:   newNames[i],
			Offset: uint64(i + 1),
		})
	}

	return dirents, nil
}

// Readlink implements p9.File.Readlink.
//
// Not properly implemented.
func (l *Local) Readlink() (string, error) {
	return os.Readlink(l.path)
}

// Flush implements p9.File.Flush.
func (l *Local) Flush() error {
	return nil
}

// Renamed implements p9.File.Renamed.
func (l *Local) Renamed(parent p9.File, newName string) {
	l.path = path.Join(parent.(*Local).path, newName)
}

// Allocate implements p9.File.Allocate.
func (l *Local) Allocate(mode p9.AllocateMode, offset, length uint64) error {
	return syscall.Fallocate(int(l.file.Fd()), mode.ToLinux(), int64(offset), int64(length))
}
<|endoftext|>


<commit_before>package actors

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"

	"github.com/cloudfoundry/cli/cf/api/application_bits"
	"github.com/cloudfoundry/cli/cf/api/resources"
	"github.com/cloudfoundry/cli/cf/app_files"
	"github.com/cloudfoundry/cli/cf/models"
	"github.com/cloudfoundry/gofileutils/fileutils"
)

const windowsPathPrefix = `\\?\`

//go:generate counterfeiter -o fakes/fake_push_actor.go . PushActor
type PushActor interface {
	UploadApp(appGuid string, zipFile *os.File, presentFiles []resources.AppFileResource) error
	ProcessPath(dirOrZipFile string, f func(string)) error
	GatherFiles(localFiles []models.AppFileFields, appDir string, uploadDir string) ([]resources.AppFileResource, bool, error)
}

type PushActorImpl struct {
	appBitsRepo application_bits.ApplicationBitsRepository
	appfiles    app_files.AppFiles
	zipper      app_files.Zipper
}

func NewPushActor(appBitsRepo application_bits.ApplicationBitsRepository, zipper app_files.Zipper, appfiles app_files.AppFiles) PushActor {
	return PushActorImpl{
		appBitsRepo: appBitsRepo,
		appfiles:    appfiles,
		zipper:      zipper,
	}
}

func (actor PushActorImpl) ProcessPath(dirOrZipFile string, f func(string)) error {
	if !actor.zipper.IsZipFile(dirOrZipFile) {
		appDir, err := filepath.EvalSymlinks(dirOrZipFile)
		if err != nil {
			return err
		}

		if filepath.IsAbs(appDir) {
			f(appDir)
		} else {
			var absPath string
			absPath, err = filepath.Abs(appDir)
			if err != nil {
				return err
			}

			f(absPath)
		}

		return nil
	}

	tempDir, err := ioutil.TempDir("", "unzipped-app")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tempDir)

	err = actor.zipper.Unzip(dirOrZipFile, tempDir)
	if err != nil {
		return err
	}

	f(tempDir)

	return nil
}

func (actor PushActorImpl) GatherFiles(localFiles []models.AppFileFields, appDir string, uploadDir string) ([]resources.AppFileResource, bool, error) {
	appFileResource := []resources.AppFileResource{}
	for _, file := range localFiles {
		appFileResource = append(appFileResource, resources.AppFileResource{
			Path: file.Path,
			Sha1: file.Sha1,
			Size: file.Size,
		})
	}

	remoteFiles, err := actor.appBitsRepo.GetApplicationFiles(appFileResource)
	if err != nil {
		return []resources.AppFileResource{}, false, err
	}

	filesToUpload := make([]models.AppFileFields, len(localFiles), len(localFiles))
	copy(filesToUpload, localFiles)

	for _, remoteFile := range remoteFiles {
		for i, fileToUpload := range filesToUpload {
			if remoteFile.Path == fileToUpload.Path {
				filesToUpload = append(filesToUpload[:i], filesToUpload[i+1:]...)
			}
		}
	}

	err = actor.appfiles.CopyFiles(filesToUpload, appDir, uploadDir)
	if err != nil {
		return []resources.AppFileResource{}, false, err
	}

	_, err = os.Stat(filepath.Join(appDir, ".cfignore"))
	if err == nil {
		err = fileutils.CopyPathToPath(filepath.Join(appDir, ".cfignore"), filepath.Join(uploadDir, ".cfignore"))
		if err != nil {
			return []resources.AppFileResource{}, false, err
		}
	}

	for i := range remoteFiles {
		fullPath, err := filepath.Abs(filepath.Join(appDir, remoteFiles[i].Path))
		if err != nil {
			return []resources.AppFileResource{}, false, err
		}

		if runtime.GOOS == "windows" {
			fullPath = windowsPathPrefix + fullPath
		}
		fileInfo, err := os.Lstat(fullPath)
		if err != nil {
			return []resources.AppFileResource{}, false, err
		}
		fileMode := fileInfo.Mode()

		if runtime.GOOS == "windows" {
			fileMode = fileMode | 0700
		}

		remoteFiles[i].Mode = fmt.Sprintf("%#o", fileMode)
	}

	return remoteFiles, len(filesToUpload) > 0, nil
}

func (actor PushActorImpl) UploadApp(appGuid string, zipFile *os.File, presentFiles []resources.AppFileResource) error {
	return actor.appBitsRepo.UploadBits(appGuid, zipFile, presentFiles)
}
<commit_msg>Add explanatory comment to ProcessPath<commit_after>package actors

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"

	"github.com/cloudfoundry/cli/cf/api/application_bits"
	"github.com/cloudfoundry/cli/cf/api/resources"
	"github.com/cloudfoundry/cli/cf/app_files"
	"github.com/cloudfoundry/cli/cf/models"
	"github.com/cloudfoundry/gofileutils/fileutils"
)

const windowsPathPrefix = `\\?\`

//go:generate counterfeiter -o fakes/fake_push_actor.go . PushActor
type PushActor interface {
	UploadApp(appGuid string, zipFile *os.File, presentFiles []resources.AppFileResource) error
	ProcessPath(dirOrZipFile string, f func(string)) error
	GatherFiles(localFiles []models.AppFileFields, appDir string, uploadDir string) ([]resources.AppFileResource, bool, error)
}

type PushActorImpl struct {
	appBitsRepo application_bits.ApplicationBitsRepository
	appfiles    app_files.AppFiles
	zipper      app_files.Zipper
}

func NewPushActor(appBitsRepo application_bits.ApplicationBitsRepository, zipper app_files.Zipper, appfiles app_files.AppFiles) PushActor {
	return PushActorImpl{
		appBitsRepo: appBitsRepo,
		appfiles:    appfiles,
		zipper:      zipper,
	}
}

// ProcessPath takes in a director of app files or a zip file which contains
// the app files. If given a zip file, it will extract the zip to a temporary
// location, call the provided callback with that location, and then clean up
// the location after the callback has been executed.
//
// This was done so that the caller of ProcessPath wouldn't need to know if it
// was a zip file or an app dir that it was given, and the caller would not be
// responsible for cleaning up the temporary directory ProcessPath creates when
// given a zip.
func (actor PushActorImpl) ProcessPath(dirOrZipFile string, f func(string)) error {
	if !actor.zipper.IsZipFile(dirOrZipFile) {
		appDir, err := filepath.EvalSymlinks(dirOrZipFile)
		if err != nil {
			return err
		}

		if filepath.IsAbs(appDir) {
			f(appDir)
		} else {
			var absPath string
			absPath, err = filepath.Abs(appDir)
			if err != nil {
				return err
			}

			f(absPath)
		}

		return nil
	}

	tempDir, err := ioutil.TempDir("", "unzipped-app")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tempDir)

	err = actor.zipper.Unzip(dirOrZipFile, tempDir)
	if err != nil {
		return err
	}

	f(tempDir)

	return nil
}

func (actor PushActorImpl) GatherFiles(localFiles []models.AppFileFields, appDir string, uploadDir string) ([]resources.AppFileResource, bool, error) {
	appFileResource := []resources.AppFileResource{}
	for _, file := range localFiles {
		appFileResource = append(appFileResource, resources.AppFileResource{
			Path: file.Path,
			Sha1: file.Sha1,
			Size: file.Size,
		})
	}

	remoteFiles, err := actor.appBitsRepo.GetApplicationFiles(appFileResource)
	if err != nil {
		return []resources.AppFileResource{}, false, err
	}

	filesToUpload := make([]models.AppFileFields, len(localFiles), len(localFiles))
	copy(filesToUpload, localFiles)

	for _, remoteFile := range remoteFiles {
		for i, fileToUpload := range filesToUpload {
			if remoteFile.Path == fileToUpload.Path {
				filesToUpload = append(filesToUpload[:i], filesToUpload[i+1:]...)
			}
		}
	}

	err = actor.appfiles.CopyFiles(filesToUpload, appDir, uploadDir)
	if err != nil {
		return []resources.AppFileResource{}, false, err
	}

	_, err = os.Stat(filepath.Join(appDir, ".cfignore"))
	if err == nil {
		err = fileutils.CopyPathToPath(filepath.Join(appDir, ".cfignore"), filepath.Join(uploadDir, ".cfignore"))
		if err != nil {
			return []resources.AppFileResource{}, false, err
		}
	}

	for i := range remoteFiles {
		fullPath, err := filepath.Abs(filepath.Join(appDir, remoteFiles[i].Path))
		if err != nil {
			return []resources.AppFileResource{}, false, err
		}

		if runtime.GOOS == "windows" {
			fullPath = windowsPathPrefix + fullPath
		}
		fileInfo, err := os.Lstat(fullPath)
		if err != nil {
			return []resources.AppFileResource{}, false, err
		}
		fileMode := fileInfo.Mode()

		if runtime.GOOS == "windows" {
			fileMode = fileMode | 0700
		}

		remoteFiles[i].Mode = fmt.Sprintf("%#o", fileMode)
	}

	return remoteFiles, len(filesToUpload) > 0, nil
}

func (actor PushActorImpl) UploadApp(appGuid string, zipFile *os.File, presentFiles []resources.AppFileResource) error {
	return actor.appBitsRepo.UploadBits(appGuid, zipFile, presentFiles)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"io"
	"regexp"
	"strings"
	"flag"
	"time"
	"compress/gzip"
)

const TIME_LAYOUT string = "[2006-01-02 15:04:05 MST]"

func main() {
	job_flag := flag.Int("jobs", 0, "Show background jobs")
	sql_flag := flag.Int("sql", 0, "Show SQL statements")
	after_str := flag.String("after", "", "Show logs after this time (YYYY-MM-DD HH:II::SS")
	match_str := flag.String("match", "", "Regexp for requests to gather")

	flag.Parse()
	args := flag.Args()

	// Time layouts must use the
	// reference time `Mon Jan 2 15:04:05 MST 2006` to show the
	// pattern with which to format/parse a given time/string

	time_after, e    := time.Parse(TIME_LAYOUT, fmt.Sprintf("[%s UTC]", *after_str))
	parse_time       := false

	if e != nil {
		if len(*after_str) > 0 {
			fmt.Println(fmt.Sprintf("Invalid time format \"%s\" - Must be YYYY-MM-DD HH::II::SS", *after_str))
			os.Exit(4)
		}
	} else {
		parse_time = true
	}

	if len(args) < 1 {
		usage()
		os.Exit(1)
	}

	fmt.Println(fmt.Sprintf("Show background jobs: %d", *job_flag))
	fmt.Println(fmt.Sprintf("Show SQL: %d", *sql_flag))
	fmt.Println(fmt.Sprintf("After: %s", *after_str))

	filename := args[0]
	fmt.Println(fmt.Sprintf("Opening file: %s", filename))

	file := openFile(filename)
	defer file.Close()

	var reader io.Reader = file

	if isGzip(filename) {
		parse_gz_reader := getGzipReader(file)
		defer parse_gz_reader.Close()

		reader = parse_gz_reader
	}

	timestamp_regexp := regexp.MustCompile("^(\\[[0-9-]+ [0-9:]+ UTC\\])")
	sql_regexp       := regexp.MustCompile("(SQL \\()|(EXEC sp_executesql N)|( CACHE \\()")
	nltm_regexp      := regexp.MustCompile(" \\(NTLM\\) ")
	request_regexp   := regexp.MustCompile("\\] (P[0-9]+[A-Za-z]+[0-9]+) ")

	var unique_map map[string]bool;

	line_count  := 0
	line_after  := !parse_time // if not parsing time, then all lines are valid
	line_strexp := *match_str
	request_ids := make([]string, 0)

	if line_regexp, err := regexp.Compile(line_strexp); len(line_strexp) > 0 && err == nil {
		scanner := bufio.NewScanner(reader);

		for scanner.Scan() {
			line := scanner.Text();
			if line_regexp.MatchString(line) {

				if !line_after {
					if timestamp := timestamp_regexp.FindStringSubmatch(line); len(timestamp) > 1 {
						if is_after_time(&timestamp[1], &time_after) {
							line_after = true
						}
					}
				}

				if line_after {
					request := request_regexp.FindStringSubmatch(line)

					if len(request) > 1 {
						is_job := strings.Contains(request[1], "DJ")

						if is_job {
							if *job_flag > 0 {
								request_ids = append(request_ids, request[1])
							} else {
								//fmt.Println(line)
							}
						} else {
							request_ids = append(request_ids, request[1])
						}
					}
				}
			}

			if line_count++; line_count % 10000 == 0 {
				fmt.Print(fmt.Sprintf("Reading: %d\r", line_count))
			}
		}

		fmt.Println("") // empty line

		if err := scanner.Err(); err != nil {
			log.Fatal(err)
		}

		fmt.Println(fmt.Sprintf("Found %d lines matching \"%s\"", len(request_ids), line_strexp))
		unique_map = generateRequestIdMap(&request_ids)

		if len(unique_map) < 1 {
			fmt.Println(fmt.Sprintf("Found 0 request identifiers", line_strexp))
			os.Exit(2)
		}

		file.Seek(0, 0)  // go back to the top (rewind)
	} else {
		fmt.Println("No matchers provided, skipping match phase")
	}

	if isGzip(filename) {
		output_gz_reader := getGzipReader(file)
		defer output_gz_reader.Close()

		reader = output_gz_reader
	}

	line_count = 0
	line_after = !parse_time // if not parsing time, then all lines are valid

	output_scanner := bufio.NewScanner(reader);

	for output_scanner.Scan() {
		line := output_scanner.Text();

		output := false

		if !line_after {
			if line_count++; line_count % 10000 == 0 {
				fmt.Print(fmt.Sprintf("Reading: %d\r", line_count))
			}

			if timestamp := timestamp_regexp.FindStringSubmatch(line); len(timestamp) > 1 {
				if is_after_time(&timestamp[1], &time_after) {
					fmt.Println("\n") // empty line
					line_after = true
				}
			}
		}

		if line_after {
			if request_id := request_regexp.FindStringSubmatch(line); len(request_id) > 1 {
				output = len(request_id[1]) > 0 && unique_map[request_id[1]]
			}
		}

		if output {
			if *sql_flag < 1 && sql_regexp.MatchString(line) {
				output = false
			} else if nltm_regexp.MatchString(line) {
				output = false
			}
		}

		if output {
			fmt.Println(line)
		}
	}

	if err := output_scanner.Err(); err != nil {
		log.Fatal(err)
	}
}

func usage() {
	fmt.Println(fmt.Sprintf("Usage: avmlog -match=\"regexp\" -jobs=0|1 -sql=0|1 -after=\"YYYY-MM-DD HH:II::SS\" avmanager_filename.log"))
	fmt.Println("Example: avm -match=\"username|computername\" \"/path/to/manager/log/production.log\"")
}

func is_after_time(timestamp *string, time_after *time.Time) bool {
	if line_time, e := time.Parse(TIME_LAYOUT, *timestamp); e != nil {
		fmt.Println("Got error %s", e)
		return false
	} else if line_time.Before(*time_after) {
		return false
	}

	return true
}

func generateRequestIdMap(request_ids *[]string) map[string]bool {
	unique_map := make(map[string]bool, len(*request_ids))

	for _, x := range *request_ids {
		unique_map[x] = true
	}

	for k, _ := range unique_map {
		fmt.Println(fmt.Sprintf("Request ID: %s", k))
	}

	return unique_map
}

func openFile(filename string) *os.File {
	file, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}
	return file
}

func isGzip(filename string) bool {
	return strings.HasSuffix(filename, ".gz")
}

func getGzipReader(file *os.File) *gzip.Reader {
	gz_reader, err := gzip.NewReader(file)
	if err != nil {
	log.Fatal(err)
	}

	return gz_reader
}<commit_msg>Move line_count and line_after variables into the if for parse - re-declare them when used for output<commit_after>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"io"
	"regexp"
	"strings"
	"flag"
	"time"
	"compress/gzip"
)

const TIME_LAYOUT string = "[2006-01-02 15:04:05 MST]"

func main() {
	job_flag := flag.Int("jobs", 0, "Show background jobs")
	sql_flag := flag.Int("sql", 0, "Show SQL statements")
	after_str := flag.String("after", "", "Show logs after this time (YYYY-MM-DD HH:II::SS")
	match_str := flag.String("match", "", "Regexp for requests to gather")

	flag.Parse()
	args := flag.Args()

	// Time layouts must use the
	// reference time `Mon Jan 2 15:04:05 MST 2006` to show the
	// pattern with which to format/parse a given time/string

	time_after, e    := time.Parse(TIME_LAYOUT, fmt.Sprintf("[%s UTC]", *after_str))
	parse_time       := false

	if e != nil {
		if len(*after_str) > 0 {
			fmt.Println(fmt.Sprintf("Invalid time format \"%s\" - Must be YYYY-MM-DD HH::II::SS", *after_str))
			os.Exit(4)
		}
	} else {
		parse_time = true
	}

	if len(args) < 1 {
		usage()
		os.Exit(1)
	}

	fmt.Println(fmt.Sprintf("Show background jobs: %d", *job_flag))
	fmt.Println(fmt.Sprintf("Show SQL: %d", *sql_flag))
	fmt.Println(fmt.Sprintf("After: %s", *after_str))

	filename := args[0]
	fmt.Println(fmt.Sprintf("Opening file: %s", filename))

	file := openFile(filename)
	defer file.Close()

	var reader io.Reader = file

	if isGzip(filename) {
		parse_gz_reader := getGzipReader(file)
		defer parse_gz_reader.Close()

		reader = parse_gz_reader
	}

	timestamp_regexp := regexp.MustCompile("^(\\[[0-9-]+ [0-9:]+ UTC\\])")
	sql_regexp       := regexp.MustCompile("(SQL \\()|(EXEC sp_executesql N)|( CACHE \\()")
	nltm_regexp      := regexp.MustCompile(" \\(NTLM\\) ")
	request_regexp   := regexp.MustCompile("\\] (P[0-9]+[A-Za-z]+[0-9]+) ")

	var unique_map map[string]bool;

	line_strexp := *match_str

	if line_regexp, err := regexp.Compile(line_strexp); len(line_strexp) > 0 && err == nil {
		line_count  := 0
		line_after  := !parse_time // if not parsing time, then all lines are valid
		request_ids := make([]string, 0)

		scanner := bufio.NewScanner(reader);

		for scanner.Scan() {
			line := scanner.Text();
			if line_regexp.MatchString(line) {

				if !line_after {
					if timestamp := timestamp_regexp.FindStringSubmatch(line); len(timestamp) > 1 {
						if is_after_time(&timestamp[1], &time_after) {
							line_after = true
						}
					}
				}

				if line_after {
					request := request_regexp.FindStringSubmatch(line)

					if len(request) > 1 {
						is_job := strings.Contains(request[1], "DJ")

						if is_job {
							if *job_flag > 0 {
								request_ids = append(request_ids, request[1])
							} else {
								//fmt.Println(line)
							}
						} else {
							request_ids = append(request_ids, request[1])
						}
					}
				}
			}

			if line_count++; line_count % 10000 == 0 {
				fmt.Print(fmt.Sprintf("Reading: %d\r", line_count))
			}
		}

		fmt.Println("") // empty line

		if err := scanner.Err(); err != nil {
			log.Fatal(err)
		}

		fmt.Println(fmt.Sprintf("Found %d lines matching \"%s\"", len(request_ids), line_strexp))
		unique_map = generateRequestIdMap(&request_ids)

		if len(unique_map) < 1 {
			fmt.Println(fmt.Sprintf("Found 0 request identifiers", line_strexp))
			os.Exit(2)
		}

		file.Seek(0, 0)  // go back to the top (rewind)
	} else {
		fmt.Println("No matchers provided, skipping match phase")
	}

	if isGzip(filename) {
		output_gz_reader := getGzipReader(file)
		defer output_gz_reader.Close()

		reader = output_gz_reader
	}

	line_count := 0
	line_after := !parse_time // if not parsing time, then all lines are valid

	output_scanner := bufio.NewScanner(reader);

	for output_scanner.Scan() {
		line := output_scanner.Text();

		output := false

		if !line_after {
			if line_count++; line_count % 10000 == 0 {
				fmt.Print(fmt.Sprintf("Reading: %d\r", line_count))
			}

			if timestamp := timestamp_regexp.FindStringSubmatch(line); len(timestamp) > 1 {
				if is_after_time(&timestamp[1], &time_after) {
					fmt.Println("\n") // empty line
					line_after = true
				}
			}
		}

		if line_after {
			if request_id := request_regexp.FindStringSubmatch(line); len(request_id) > 1 {
				output = len(request_id[1]) > 0 && unique_map[request_id[1]]
			}
		}

		if output {
			if *sql_flag < 1 && sql_regexp.MatchString(line) {
				output = false
			} else if nltm_regexp.MatchString(line) {
				output = false
			}
		}

		if output {
			fmt.Println(line)
		}
	}

	if err := output_scanner.Err(); err != nil {
		log.Fatal(err)
	}
}

func usage() {
	fmt.Println(fmt.Sprintf("Usage: avmlog -match=\"regexp\" -jobs=0|1 -sql=0|1 -after=\"YYYY-MM-DD HH:II::SS\" avmanager_filename.log"))
	fmt.Println("Example: avm -match=\"username|computername\" \"/path/to/manager/log/production.log\"")
}

func is_after_time(timestamp *string, time_after *time.Time) bool {
	if line_time, e := time.Parse(TIME_LAYOUT, *timestamp); e != nil {
		fmt.Println("Got error %s", e)
		return false
	} else if line_time.Before(*time_after) {
		return false
	}

	return true
}

func generateRequestIdMap(request_ids *[]string) map[string]bool {
	unique_map := make(map[string]bool, len(*request_ids))

	for _, x := range *request_ids {
		unique_map[x] = true
	}

	for k, _ := range unique_map {
		fmt.Println(fmt.Sprintf("Request ID: %s", k))
	}

	return unique_map
}

func openFile(filename string) *os.File {
	file, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}
	return file
}

func isGzip(filename string) bool {
	return strings.HasSuffix(filename, ".gz")
}

func getGzipReader(file *os.File) *gzip.Reader {
	gz_reader, err := gzip.NewReader(file)
	if err != nil {
	log.Fatal(err)
	}

	return gz_reader
}<|endoftext|>


<commit_before>package main

import "testing"

func testIcons_Find(terms []string) Icons {
	return NewIcons().Find(terms)
}

func TestIcons_Find_AllIcons(t *testing.T) {
	fi := testIcons_Find([]string{""})

	actual := len(fi)
	expected := 519
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_Find_ZeroIcon(t *testing.T) {
	fi := testIcons_Find([]string{"foo-bar-baz"})

	actual := len(fi)
	expected := 0
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_Find_OneIcon(t *testing.T) {
	fi := testIcons_Find([]string{"github-square"})

	actual := len(fi)
	expected := 1
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_Find_TwoIcons(t *testing.T) {
	fi := testIcons_Find([]string{"github-"})

	actual := len(fi)
	expected := 2
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_Find_FirstIcon(t *testing.T) {
	fi := testIcons_Find([]string{""})

	actual := fi[0].ID
	expected := "adjust"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_Find_LastIcon(t *testing.T) {
	fi := testIcons_Find([]string{""})

	actual := fi[len(fi)-1].ID
	expected := "youtube-square"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_Find_TaxiIcon(t *testing.T) {
	fi := testIcons_Find([]string{"taxi"})

	actual := fi[0].Name
	expected := "Taxi"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].ID
	expected = "taxi"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Unicode
	expected = "f1ba"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Created
	expected = "4.1"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Aliases[0]
	expected = "cab"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Filter[0]
	expected = "vehicle"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Categories[0]
	expected = "Web Application Icons"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_Find_Aliases(t *testing.T) {
	fi := testIcons_Find([]string{"navicon"})

	actual := fi[0].ID
	expected := "bars"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	if len(fi) != 1 {
		t.Errorf("expected %v to eq %v", len(fi), 1)
	}
}
<commit_msg>Rename helper function<commit_after>package main

import "testing"

func testIconsHelper_Find(terms []string) Icons {
	return NewIcons().Find(terms)
}

func TestIcons_Find_AllIcons(t *testing.T) {
	fi := testIconsHelper_Find([]string{""})

	actual := len(fi)
	expected := 519
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_Find_ZeroIcon(t *testing.T) {
	fi := testIconsHelper_Find([]string{"foo-bar-baz"})

	actual := len(fi)
	expected := 0
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_Find_OneIcon(t *testing.T) {
	fi := testIconsHelper_Find([]string{"github-square"})

	actual := len(fi)
	expected := 1
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_Find_TwoIcons(t *testing.T) {
	fi := testIconsHelper_Find([]string{"github-"})

	actual := len(fi)
	expected := 2
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_Find_FirstIcon(t *testing.T) {
	fi := testIconsHelper_Find([]string{""})

	actual := fi[0].ID
	expected := "adjust"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_Find_LastIcon(t *testing.T) {
	fi := testIconsHelper_Find([]string{""})

	actual := fi[len(fi)-1].ID
	expected := "youtube-square"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_Find_TaxiIcon(t *testing.T) {
	fi := testIconsHelper_Find([]string{"taxi"})

	actual := fi[0].Name
	expected := "Taxi"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].ID
	expected = "taxi"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Unicode
	expected = "f1ba"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Created
	expected = "4.1"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Aliases[0]
	expected = "cab"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Filter[0]
	expected = "vehicle"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Categories[0]
	expected = "Web Application Icons"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_Find_Aliases(t *testing.T) {
	fi := testIconsHelper_Find([]string{"navicon"})

	actual := fi[0].ID
	expected := "bars"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	if len(fi) != 1 {
		t.Errorf("expected %v to eq %v", len(fi), 1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/gosuri/uiprogress"
	"github.com/gosuri/uiprogress/util/strutil"

	"gopkg.in/kothar/go-backblaze.v0"
)

// TODO support directories
// TODO support replacing all previous versions
type Put struct {
}

func init() {
	parser.AddCommand("put", "Store a file",
		"Uploads one or more files. Specify the bucket with -b, and the filenames to upload as extra arguments.",
		&Put{})
}

func (o *Put) Execute(args []string) error {
	client, err := Client()
	if err != nil {
		return err
	}

	bucket, err := client.Bucket(opts.Bucket)
	if err != nil {
		return err
	}
	if bucket == nil {
		return errors.New("Bucket not found: " + opts.Bucket)
	}

	uiprogress.Start()
	for _, file := range args {
		_, err := upload(bucket, file)
		if err != nil {
			return err
		}
	}

	return nil
}

type progressReader struct {
	bar *uiprogress.Bar
	r   io.ReadSeeker
}

func (p *progressReader) Read(b []byte) (int, error) {
	read, err := p.r.Read(b)
	p.bar.Set(p.bar.Current() + read)
	return read, err
}

func (p *progressReader) Seek(offset int64, whence int) (int64, error) {
	switch whence {
	case 0:
		p.bar.Set(int(offset))
	case 1:
		p.bar.Set(p.bar.Current() + int(offset))
	case 2:
		p.bar.Set(p.bar.Total - int(offset))
	}
	return p.r.Seek(offset, whence)
}

func upload(bucket *backblaze.Bucket, file string) (*backblaze.File, error) {

	stat, err := os.Stat(file)
	if err != nil {
		return nil, err
	}

	reader, err := os.Open(file)
	if err != nil {
		return nil, err
	}
	defer reader.Close()

	var r io.Reader = reader
	if opts.Verbose {
		bar := uiprogress.AddBar(int(stat.Size()))
		bar.AppendFunc(func(b *uiprogress.Bar) string {
			speed := (float32(b.Current()) / 1024) / float32(b.TimeElapsed().Seconds())
			return fmt.Sprintf("%7.2f KB/s", speed)
		})
		bar.AppendCompleted()
		bar.PrependFunc(func(b *uiprogress.Bar) string { return fmt.Sprintf("%10d", b.Total) })
		bar.PrependFunc(func(b *uiprogress.Bar) string { return strutil.Resize(file, 50) })
		bar.Width = 20

		r = &progressReader{bar, reader}
	}

	return bucket.UploadFile(filepath.Base(file), nil, r)
}
<commit_msg>Add metadata support to b2 put command<commit_after>package main

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/gosuri/uiprogress"
	"github.com/gosuri/uiprogress/util/strutil"

	"gopkg.in/kothar/go-backblaze.v0"
)

// TODO support directories
// TODO support replacing all previous versions
type Put struct {
	Meta map[string]string `long:"meta" description:"Assign metadata to uploaded files"`
}

func init() {
	parser.AddCommand("put", "Store a file",
		"Uploads one or more files. Specify the bucket with -b, and the filenames to upload as extra arguments.",
		&Put{})
}

func (o *Put) Execute(args []string) error {
	client, err := Client()
	if err != nil {
		return err
	}

	bucket, err := client.Bucket(opts.Bucket)
	if err != nil {
		return err
	}
	if bucket == nil {
		return errors.New("Bucket not found: " + opts.Bucket)
	}

	uiprogress.Start()
	for _, file := range args {
		_, err := upload(bucket, file, o.Meta)
		if err != nil {
			return err
		}
	}

	return nil
}

type progressReader struct {
	bar *uiprogress.Bar
	r   io.ReadSeeker
}

func (p *progressReader) Read(b []byte) (int, error) {
	read, err := p.r.Read(b)
	p.bar.Set(p.bar.Current() + read)
	return read, err
}

func (p *progressReader) Seek(offset int64, whence int) (int64, error) {
	switch whence {
	case 0:
		p.bar.Set(int(offset))
	case 1:
		p.bar.Set(p.bar.Current() + int(offset))
	case 2:
		p.bar.Set(p.bar.Total - int(offset))
	}
	return p.r.Seek(offset, whence)
}

func upload(bucket *backblaze.Bucket, file string, meta map[string]string) (*backblaze.File, error) {

	stat, err := os.Stat(file)
	if err != nil {
		return nil, err
	}

	reader, err := os.Open(file)
	if err != nil {
		return nil, err
	}
	defer reader.Close()

	var r io.Reader = reader
	if opts.Verbose {
		bar := uiprogress.AddBar(int(stat.Size()))
		bar.AppendFunc(func(b *uiprogress.Bar) string {
			speed := (float32(b.Current()) / 1024) / float32(b.TimeElapsed().Seconds())
			return fmt.Sprintf("%7.2f KB/s", speed)
		})
		bar.AppendCompleted()
		bar.PrependFunc(func(b *uiprogress.Bar) string { return fmt.Sprintf("%10d", b.Total) })
		bar.PrependFunc(func(b *uiprogress.Bar) string { return strutil.Resize(file, 50) })
		bar.Width = 20

		r = &progressReader{bar, reader}
	}

	return bucket.UploadFile(filepath.Base(file), meta, r)
}
<|endoftext|>


<commit_before>// Package logging provides a fast, asynchronous request logger which outputs
// NCSA/Apache combined logs.
package logging

import (
	"fmt"
	"io"
	"net/http"
	"strings"
	"time"
)

// A LoggingHandler is a HTTP handler which proxies requests to an underlying
// handler and logs the results.
type LoggingHandler struct {
	clock   clock
	w       io.Writer
	handler http.Handler
	buffer  chan string
	quit    chan struct{}
}

// Wrap returns the underlying handler, wrapped in a LoggingHandler which will
// write to the given Writer. N.B.: You must call Start() on the result before
// using it.
func Wrap(h http.Handler, w io.Writer) *LoggingHandler {
	return &LoggingHandler{
		clock:   time.Now,
		w:       w,
		handler: h,
		buffer:  make(chan string, 1000),
		quit:    make(chan struct{}),
	}
}

// Start creates a goroutine to handle the logging IO.
func (al *LoggingHandler) Start() {
	go func() {
		for s := range al.buffer {
			fmt.Fprint(al.w, s)
		}
		close(al.quit)
	}()
}

// Stop closes the internal channel used to buffer log statements and waits for
// the IO goroutine to complete.
func (al *LoggingHandler) Stop() {
	close(al.buffer)
	<-al.quit

}

func (al *LoggingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	wrapper := &responseWrapper{w: w, status: 200}

	start := al.clock()
	al.handler.ServeHTTP(wrapper, r)
	end := al.clock()

	remoteAddr := r.RemoteAddr
	if index := strings.LastIndex(remoteAddr, ":"); index != -1 {
		remoteAddr = remoteAddr[:index]
	}

	if s := r.Header.Get(xForwardedFor); s != "" {
		remoteAddr = s
	}

	referer := r.Referer()
	if "" == referer {
		referer = "-"
	}

	userAgent := r.UserAgent()
	if "" == userAgent {
		userAgent = "-"
	}

	al.buffer <- fmt.Sprintf(
		"%s %s %s [%s] \"%s %s %s\" %d %d %q %q %d %q\n",
		remoteAddr,
		"-", // We're not supporting identd, sorry.
		"-", // We're also not supporting basic auth.
		start.In(time.UTC).Format(apacheFormat),
		r.Method,
		r.RequestURI,
		r.Proto,
		wrapper.status,
		0,
		referer,
		userAgent,
		end.Sub(start).Nanoseconds()/int64(time.Millisecond),
		r.Header.Get(xRequestID),
	)
}

const (
	apacheFormat  = "02/Jan/2006:15:04:05 -0700"
	xRequestID    = "X-Request-Id"
	xForwardedFor = "X-Forwarded-For"
)

type responseWrapper struct {
	w      http.ResponseWriter
	status int
}

func (w *responseWrapper) Header() http.Header {
	return w.w.Header()
}

func (w *responseWrapper) Write(b []byte) (int, error) {
	return w.w.Write(b)
}

func (w *responseWrapper) WriteHeader(status int) {
	w.status = status
	w.w.WriteHeader(status)
}

type clock func() time.Time
<commit_msg>Delete blank line<commit_after>// Package logging provides a fast, asynchronous request logger which outputs
// NCSA/Apache combined logs.
package logging

import (
	"fmt"
	"io"
	"net/http"
	"strings"
	"time"
)

// A LoggingHandler is a HTTP handler which proxies requests to an underlying
// handler and logs the results.
type LoggingHandler struct {
	clock   clock
	w       io.Writer
	handler http.Handler
	buffer  chan string
	quit    chan struct{}
}

// Wrap returns the underlying handler, wrapped in a LoggingHandler which will
// write to the given Writer. N.B.: You must call Start() on the result before
// using it.
func Wrap(h http.Handler, w io.Writer) *LoggingHandler {
	return &LoggingHandler{
		clock:   time.Now,
		w:       w,
		handler: h,
		buffer:  make(chan string, 1000),
		quit:    make(chan struct{}),
	}
}

// Start creates a goroutine to handle the logging IO.
func (al *LoggingHandler) Start() {
	go func() {
		for s := range al.buffer {
			fmt.Fprint(al.w, s)
		}
		close(al.quit)
	}()
}

// Stop closes the internal channel used to buffer log statements and waits for
// the IO goroutine to complete.
func (al *LoggingHandler) Stop() {
	close(al.buffer)
	<-al.quit
}

func (al *LoggingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	wrapper := &responseWrapper{w: w, status: 200}

	start := al.clock()
	al.handler.ServeHTTP(wrapper, r)
	end := al.clock()

	remoteAddr := r.RemoteAddr
	if index := strings.LastIndex(remoteAddr, ":"); index != -1 {
		remoteAddr = remoteAddr[:index]
	}

	if s := r.Header.Get(xForwardedFor); s != "" {
		remoteAddr = s
	}

	referer := r.Referer()
	if "" == referer {
		referer = "-"
	}

	userAgent := r.UserAgent()
	if "" == userAgent {
		userAgent = "-"
	}

	al.buffer <- fmt.Sprintf(
		"%s %s %s [%s] \"%s %s %s\" %d %d %q %q %d %q\n",
		remoteAddr,
		"-", // We're not supporting identd, sorry.
		"-", // We're also not supporting basic auth.
		start.In(time.UTC).Format(apacheFormat),
		r.Method,
		r.RequestURI,
		r.Proto,
		wrapper.status,
		0,
		referer,
		userAgent,
		end.Sub(start).Nanoseconds()/int64(time.Millisecond),
		r.Header.Get(xRequestID),
	)
}

const (
	apacheFormat  = "02/Jan/2006:15:04:05 -0700"
	xRequestID    = "X-Request-Id"
	xForwardedFor = "X-Forwarded-For"
)

type responseWrapper struct {
	w      http.ResponseWriter
	status int
}

func (w *responseWrapper) Header() http.Header {
	return w.w.Header()
}

func (w *responseWrapper) Write(b []byte) (int, error) {
	return w.w.Write(b)
}

func (w *responseWrapper) WriteHeader(status int) {
	w.status = status
	w.w.WriteHeader(status)
}

type clock func() time.Time
<|endoftext|>


<commit_before>package logging

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"strings"

	"github.com/hashicorp/go-syslog"
	"github.com/hashicorp/logutils"
)

// Levels are the log levels we respond to=o.
var Levels = []logutils.LogLevel{"DEBUG", "INFO", "WARN", "ERR"}

// Config is the configuration for this log setup.
type Config struct {
	// Name is the progname as it will appear in syslog output (if enabled).
	Name string `json:"name"`

	// Level is the log level to use.
	Level string `json:"level"`

	// Syslog and SyslogFacility are the syslog configuration options.
	Syslog         bool   `json:"syslog"`
	SyslogFacility string `json:"syslog_facility"`

	// Writer is the output where logs should go. If syslog is enabled, data will
	// be written to writer in addition to syslog.
	Writer io.Writer `json:"-"`
}

func Setup(config *Config) error {
	var logOutput io.Writer

	// Setup the default logging
	logFilter := NewLogFilter()
	logFilter.MinLevel = logutils.LogLevel(strings.ToUpper(config.Level))
	logFilter.Writer = config.Writer
	if !ValidateLevelFilter(logFilter.MinLevel, logFilter) {
		levels := make([]string, 0, len(logFilter.Levels))
		for _, level := range logFilter.Levels {
			levels = append(levels, string(level))
		}
		return fmt.Errorf("invalid log level %q, valid log levels are %s",
			config.Level, strings.Join(levels, ", "))
	}

	// Check if syslog is enabled
	if config.Syslog {
		log.Printf("[DEBUG] (logging) enabling syslog on %s", config.SyslogFacility)

		l, err := gsyslog.NewLogger(gsyslog.LOG_NOTICE, config.SyslogFacility, config.Name)
		if err != nil {
			return fmt.Errorf("error setting up syslog logger: %s", err)
		}
		syslog := &SyslogWrapper{l, logFilter}
		logOutput = io.MultiWriter(logFilter, syslog)
	} else {
		logOutput = io.MultiWriter(logFilter)
	}

	log.SetOutput(logOutput)

	return nil
}

// NewLogFilter returns a LevelFilter that is configured with the log levels that
// we use.
func NewLogFilter() *logutils.LevelFilter {
	return &logutils.LevelFilter{
		Levels:   Levels,
		MinLevel: "WARN",
		Writer:   ioutil.Discard,
	}
}

// ValidateLevelFilter verifies that the log levels within the filter are valid.
func ValidateLevelFilter(min logutils.LogLevel, filter *logutils.LevelFilter) bool {
	for _, level := range filter.Levels {
		if level == min {
			return true
		}
	}
	return false
}
<commit_msg>Log in UTC and log milliseconds<commit_after>package logging

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"strings"

	"github.com/hashicorp/go-syslog"
	"github.com/hashicorp/logutils"
)

// Levels are the log levels we respond to=o.
var Levels = []logutils.LogLevel{"DEBUG", "INFO", "WARN", "ERR"}

// Config is the configuration for this log setup.
type Config struct {
	// Name is the progname as it will appear in syslog output (if enabled).
	Name string `json:"name"`

	// Level is the log level to use.
	Level string `json:"level"`

	// Syslog and SyslogFacility are the syslog configuration options.
	Syslog         bool   `json:"syslog"`
	SyslogFacility string `json:"syslog_facility"`

	// Writer is the output where logs should go. If syslog is enabled, data will
	// be written to writer in addition to syslog.
	Writer io.Writer `json:"-"`
}

func Setup(config *Config) error {
	var logOutput io.Writer

	// Setup the default logging
	logFilter := NewLogFilter()
	logFilter.MinLevel = logutils.LogLevel(strings.ToUpper(config.Level))
	logFilter.Writer = config.Writer
	if !ValidateLevelFilter(logFilter.MinLevel, logFilter) {
		levels := make([]string, 0, len(logFilter.Levels))
		for _, level := range logFilter.Levels {
			levels = append(levels, string(level))
		}
		return fmt.Errorf("invalid log level %q, valid log levels are %s",
			config.Level, strings.Join(levels, ", "))
	}

	// Check if syslog is enabled
	if config.Syslog {
		log.Printf("[DEBUG] (logging) enabling syslog on %s", config.SyslogFacility)

		l, err := gsyslog.NewLogger(gsyslog.LOG_NOTICE, config.SyslogFacility, config.Name)
		if err != nil {
			return fmt.Errorf("error setting up syslog logger: %s", err)
		}
		syslog := &SyslogWrapper{l, logFilter}
		logOutput = io.MultiWriter(logFilter, syslog)
	} else {
		logOutput = io.MultiWriter(logFilter)
	}

	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds | log.LUTC)
	log.SetOutput(logOutput)

	return nil
}

// NewLogFilter returns a LevelFilter that is configured with the log levels that
// we use.
func NewLogFilter() *logutils.LevelFilter {
	return &logutils.LevelFilter{
		Levels:   Levels,
		MinLevel: "WARN",
		Writer:   ioutil.Discard,
	}
}

// ValidateLevelFilter verifies that the log levels within the filter are valid.
func ValidateLevelFilter(min logutils.LogLevel, filter *logutils.LevelFilter) bool {
	for _, level := range filter.Levels {
		if level == min {
			return true
		}
	}
	return false
}
<|endoftext|>