<commit_before>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package remote implements an inprocess directory server that uses RPC to
// connect to a remote directory server.
package remote

import (
	"fmt"
	"io"
	"strings"

	"upspin.io/auth/grpcauth"
	"upspin.io/bind"
	"upspin.io/errors"
	"upspin.io/log"
	"upspin.io/upspin"
	"upspin.io/upspin/proto"
)

// requireAuthentication specifies whether the connection demands TLS.
const requireAuthentication = true

// dialContext contains the destination and authenticated user of the dial.
type dialContext struct {
	endpoint upspin.Endpoint
	userName upspin.UserName
}

// remote implements upspin.DirServer.
type remote struct {
	*grpcauth.AuthClientService // For handling Authenticate, Ping and Close.
	ctx                         dialContext
	dirClient                   proto.DirClient
}

var _ upspin.DirServer = (*remote)(nil)

// Glob implements upspin.DirServer.Glob.
func (r *remote) Glob(pattern string) ([]*upspin.DirEntry, error) {
	op := r.opf("Glob", "%q", pattern)

	gCtx, callOpt, finishAuth, err := r.NewAuthContext()
	if err != nil {
		return nil, op.error(err)
	}
	req := &proto.DirGlobRequest{
		Pattern: pattern,
	}
	resp, err := r.dirClient.Glob(gCtx, req, callOpt)
	err = finishAuth(err)
	if err != nil {
		r.checkGrpc(err)
		return nil, op.error(errors.IO, err)
	}

	err = unmarshalError(resp.Error)
	if err != nil && err != upspin.ErrFollowLink {
		return nil, op.error(err)
	}
	entries, pErr := proto.UpspinDirEntries(resp.Entries)
	if pErr != nil {
		return nil, op.error(errors.IO, pErr)
	}
	return entries, op.error(err)
}

func entryName(entry *upspin.DirEntry) string {
	if entry == nil {
		return "<nil>"
	}
	return fmt.Sprintf("%q", entry.Name)
}

// Put implements upspin.DirServer.Put.
func (r *remote) Put(entry *upspin.DirEntry) (*upspin.DirEntry, error) {
	op := r.opf("Put", "%s", entryName(entry))

	gCtx, callOpt, finishAuth, err := r.NewAuthContext()
	if err != nil {
		return nil, op.error(err)
	}
	b, err := entry.Marshal()
	if err != nil {
		return nil, op.error(err)
	}
	req := &proto.DirPutRequest{
		Entry: b,
	}
	resp, err := r.dirClient.Put(gCtx, req, callOpt)
	err = finishAuth(err)
	r.checkGrpc(err)
	return op.entryError(resp, err)
}

// WhichAccess implements upspin.DirServer.WhichAccess.
func (r *remote) WhichAccess(pathName upspin.PathName) (*upspin.DirEntry, error) {
	op := r.opf("WhichAccess", "%q", pathName)

	gCtx, callOpt, finishAuth, err := r.NewAuthContext()
	if err != nil {
		return nil, op.error(err)
	}
	req := &proto.DirWhichAccessRequest{
		Name: string(pathName),
	}
	resp, err := r.dirClient.WhichAccess(gCtx, req, callOpt)
	err = finishAuth(err)
	r.checkGrpc(err)
	return op.entryError(resp, err)
}

// Delete implements upspin.DirServer.Delete.
func (r *remote) Delete(pathName upspin.PathName) (*upspin.DirEntry, error) {
	op := r.opf("Delete", "%q", pathName)

	gCtx, callOpt, finishAuth, err := r.NewAuthContext()
	if err != nil {
		return nil, op.error(err)
	}
	req := &proto.DirDeleteRequest{
		Name: string(pathName),
	}
	resp, err := r.dirClient.Delete(gCtx, req, callOpt)
	err = finishAuth(err)
	r.checkGrpc(err)
	return op.entryError(resp, err)
}

// Lookup implements upspin.DirServer.Lookup.
func (r *remote) Lookup(pathName upspin.PathName) (*upspin.DirEntry, error) {
	op := r.opf("Lookup", "%q", pathName)

	gCtx, callOpt, finishAuth, err := r.NewAuthContext()
	if err != nil {
		return nil, op.error(err)
	}
	req := &proto.DirLookupRequest{
		Name: string(pathName),
	}
	resp, err := r.dirClient.Lookup(gCtx, req, callOpt)
	err = finishAuth(err)
	r.checkGrpc(err)
	return op.entryError(resp, err)
}

// Watch implements upspin.DirServer.
func (r *remote) Watch(name upspin.PathName, order int64, done <-chan struct{}) (<-chan upspin.Event, error) {
	op := r.opf("Watch", "%q %d", name, order)

	gCtx, callOpt, finishAuth, err := r.NewAuthContext()
	if err != nil {
		return nil, op.error(err)
	}
	srvStream, err := r.dirClient.Watch(gCtx, callOpt)
	err = finishAuth(err)
	if err != nil {
		return nil, op.error(err)
	}
	req := &proto.DirWatchRequest{
		Name:  string(name),
		Order: order,
	}
	err = srvStream.Send(req)
	if err != nil {
		return nil, op.error(err)
	}

	// First event back determines if Watch call was successful.
	protoEvent, err := srvStream.Recv()
	if err != nil {
		return nil, op.error(err)
	}
	event, err := proto.UpspinEvent(protoEvent)
	if err != nil {
		return nil, op.error(err)
	}
	if event.Error != nil {
		return nil, event.Error
	}
	// Assert event.Entry is nil, since this is a synthetic event, which
	// serves to confirm whether Watch succeeded or not.
	if event.Entry != nil {
		return nil, op.error(errors.Internal, errors.Str("first event must have nil entry"))
	}
	// No error on Watch. Establish output channel.
	events := make(chan upspin.Event, 1)
	serverDone := make(chan struct{})
	go r.watchDone(op, srvStream, events, done, serverDone)
	go r.watch(op, srvStream, events, serverDone)
	return events, nil
}

// watchDone, which runs in a goroutine, waits for either the client to close
// its done channel or the server to close its Events channel. When either one
// happens it closes the send-side GRPC stream, which in turn signals the server
// to finish up on its side.
func (r *remote) watchDone(op *operation, srvStream proto.Dir_WatchClient, events chan upspin.Event, done, serverDone <-chan struct{}) {
	select {
	case <-done:
		op.logf("Client closed done channel")
	case <-serverDone:
		op.logf("Server closed events channel")
	}
	srvStream.CloseSend()
}

// watch, which runs in a goroutine, receives events from the server and relays
// them to the client onto the events channel. When the server terminates the
// stream (sends the error io.EOF), we close the events channel and notify the
// other goroutine that we've terminated.
//
// Note: there's no need for the sending of events to time out since this is
// the client side. It can hang forever. But it probably won't because all other
// layers have timeouts.
func (r *remote) watch(op *operation, srvStream proto.Dir_WatchClient, events chan upspin.Event, serverDone chan struct{}) {
	defer close(events)
	defer close(serverDone) // tell the other goroutine the server is done.

	for {
		protoEvent, err := srvStream.Recv()
		if err == io.EOF {
			// Server closed the channel. Nothing else to do but to
			// close ours too.
			return
		}
		if err != nil {
			op.logErr(err)
			events <- upspin.Event{Error: err}
			return
		}
		event, err := proto.UpspinEvent(protoEvent)
		if err != nil {
			op.logErr(err)
			events <- upspin.Event{Error: err}
			return
		}
		events <- *event
	}
}

// Endpoint implements upspin.StoreServer.Endpoint.
func (r *remote) Endpoint() upspin.Endpoint {
	return r.ctx.endpoint
}

func dialCache(op *operation, context upspin.Context, proxyFor upspin.Endpoint) upspin.Service {
	// Are we using a cache?
	ce := context.CacheEndpoint()
	if ce.Transport == upspin.Unassigned {
		return nil
	}

	// Call the cache. The cache is local so don't bother with TLS.
	authClient, err := grpcauth.NewGRPCClient(context, ce.NetAddr, grpcauth.KeepAliveInterval, grpcauth.NoSecurity, proxyFor)
	if err != nil {
		// On error dial direct.
		op.error(errors.IO, err)
		return nil
	}

	// The connection is closed when this service is released (see Bind.Release).
	dirClient := proto.NewDirClient(authClient.GRPCConn())
	authClient.SetService(dirClient)

	return &remote{
		AuthClientService: authClient,
		ctx: dialContext{
			endpoint: proxyFor,
			userName: context.UserName(),
		},
		dirClient: dirClient,
	}
}

// Dial implements upspin.Service.
func (r *remote) Dial(context upspin.Context, e upspin.Endpoint) (upspin.Service, error) {
	op := r.opf("Dial", "%q, %q", context.UserName(), e)

	if e.Transport != upspin.Remote {
		return nil, op.error(errors.Invalid, errors.Str("unrecognized transport"))
	}

	// First try a cache
	if svc := dialCache(op, context, e); svc != nil {
		return svc, nil
	}

	authClient, err := grpcauth.NewGRPCClient(context, e.NetAddr, grpcauth.KeepAliveInterval, grpcauth.Secure, upspin.Endpoint{})
	if err != nil {
		return nil, op.error(errors.IO, err)
	}

	// The connection is closed when this service is released (see Bind.Release)
	dirClient := proto.NewDirClient(authClient.GRPCConn())
	authClient.SetService(dirClient)
	r = &remote{
		AuthClientService: authClient,
		ctx: dialContext{
			endpoint: e,
			userName: context.UserName(),
		},
		dirClient: dirClient,
	}

	return r, nil
}

const transport = upspin.Remote

func init() {
	r := &remote{} // uninitialized until Dial time.
	bind.RegisterDirServer(transport, r)
}

// unmarshalError calls proto.UnmarshalError, but if the error
// matches upspin.ErrFollowLink, it returns that exact value so
// the caller can do == to test for links.
func unmarshalError(b []byte) error {
	err := errors.UnmarshalError(b)
	if err != nil && err.Error() == upspin.ErrFollowLink.Error() {
		return upspin.ErrFollowLink
	}
	return err
}

func (r *remote) opf(method string, format string, args ...interface{}) *operation {
	ep := r.ctx.endpoint.String()
	s := fmt.Sprintf("dir/remote.%s(%q)", method, ep)
	op := &operation{s, fmt.Sprintf(format, args...)}
	log.Debug.Print(op)
	return op
}

type operation struct {
	op   string
	args string
}

func (op *operation) String() string {
	return fmt.Sprintf("%s(%s)", op.op, op.args)
}

func (op *operation) logErr(err error) {
	log.Error.Printf("%s: %s", op, err)
}

func (op *operation) logf(format string, args ...interface{}) {
	log.Printf("%s: "+format, append([]interface{}{op.op}, args...)...)
}

func (op *operation) error(args ...interface{}) error {
	if len(args) == 0 {
		panic("error called with zero args")
	}
	if len(args) == 1 {
		if e, ok := args[0].(error); ok && e == upspin.ErrFollowLink {
			return e
		}
		if args[0] == nil {
			return nil
		}
	}
	log.Debug.Printf("%v error: %v", op, errors.E(args...))
	return errors.E(append([]interface{}{op.op}, args...)...)
}

// entryError performs the common operation of converting an EntryError
// protocol buffer into a directory entry and error pair.
func (op *operation) entryError(p *proto.EntryError, err error) (*upspin.DirEntry, error) {
	if err != nil {
		return nil, op.error(errors.IO, err)
	}
	err = unmarshalError(p.Error)
	if err != nil && err != upspin.ErrFollowLink {
		return nil, op.error(err)
	}
	entry, unmarshalErr := proto.UpspinDirEntry(p.Entry)
	if unmarshalErr != nil {
		return nil, op.error(unmarshalErr)
	}
	return entry, op.error(err)
}

// GRPC will call back a server if HTTP2 signals that the underlying connection has
// been terminated. However, shiould the server be unreachable, GRPC will poison
// the connection and no longer retry the connection returning a transport error on
// all subsequent RPCs. As a result we will not survive server bounces.
//
// To remedy this, we must recognize this condition and remove the service
// from bind's cache and close the GRPC connection. While grpc has an internal
// ConnectivityState called Shutdown that indicates this condition we have no
// access to it. Instead we must examine the error return from GRPC RPCs.
func (r *remote) checkGrpc(err error) {
	if err == nil || !strings.Contains(err.Error(), "transport: dial") {
		return
	}
	bind.Release(r)
}
<commit_msg>dir/remote: call checkGrpc in Watch method<commit_after>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package remote implements an inprocess directory server that uses RPC to
// connect to a remote directory server.
package remote

import (
	"fmt"
	"io"
	"strings"

	"upspin.io/auth/grpcauth"
	"upspin.io/bind"
	"upspin.io/errors"
	"upspin.io/log"
	"upspin.io/upspin"
	"upspin.io/upspin/proto"
)

// requireAuthentication specifies whether the connection demands TLS.
const requireAuthentication = true

// dialContext contains the destination and authenticated user of the dial.
type dialContext struct {
	endpoint upspin.Endpoint
	userName upspin.UserName
}

// remote implements upspin.DirServer.
type remote struct {
	*grpcauth.AuthClientService // For handling Authenticate, Ping and Close.
	ctx                         dialContext
	dirClient                   proto.DirClient
}

var _ upspin.DirServer = (*remote)(nil)

// Glob implements upspin.DirServer.Glob.
func (r *remote) Glob(pattern string) ([]*upspin.DirEntry, error) {
	op := r.opf("Glob", "%q", pattern)

	gCtx, callOpt, finishAuth, err := r.NewAuthContext()
	if err != nil {
		return nil, op.error(err)
	}
	req := &proto.DirGlobRequest{
		Pattern: pattern,
	}
	resp, err := r.dirClient.Glob(gCtx, req, callOpt)
	err = finishAuth(err)
	if err != nil {
		r.checkGrpc(err)
		return nil, op.error(errors.IO, err)
	}

	err = unmarshalError(resp.Error)
	if err != nil && err != upspin.ErrFollowLink {
		return nil, op.error(err)
	}
	entries, pErr := proto.UpspinDirEntries(resp.Entries)
	if pErr != nil {
		return nil, op.error(errors.IO, pErr)
	}
	return entries, op.error(err)
}

func entryName(entry *upspin.DirEntry) string {
	if entry == nil {
		return "<nil>"
	}
	return fmt.Sprintf("%q", entry.Name)
}

// Put implements upspin.DirServer.Put.
func (r *remote) Put(entry *upspin.DirEntry) (*upspin.DirEntry, error) {
	op := r.opf("Put", "%s", entryName(entry))

	gCtx, callOpt, finishAuth, err := r.NewAuthContext()
	if err != nil {
		return nil, op.error(err)
	}
	b, err := entry.Marshal()
	if err != nil {
		return nil, op.error(err)
	}
	req := &proto.DirPutRequest{
		Entry: b,
	}
	resp, err := r.dirClient.Put(gCtx, req, callOpt)
	err = finishAuth(err)
	r.checkGrpc(err)
	return op.entryError(resp, err)
}

// WhichAccess implements upspin.DirServer.WhichAccess.
func (r *remote) WhichAccess(pathName upspin.PathName) (*upspin.DirEntry, error) {
	op := r.opf("WhichAccess", "%q", pathName)

	gCtx, callOpt, finishAuth, err := r.NewAuthContext()
	if err != nil {
		return nil, op.error(err)
	}
	req := &proto.DirWhichAccessRequest{
		Name: string(pathName),
	}
	resp, err := r.dirClient.WhichAccess(gCtx, req, callOpt)
	err = finishAuth(err)
	r.checkGrpc(err)
	return op.entryError(resp, err)
}

// Delete implements upspin.DirServer.Delete.
func (r *remote) Delete(pathName upspin.PathName) (*upspin.DirEntry, error) {
	op := r.opf("Delete", "%q", pathName)

	gCtx, callOpt, finishAuth, err := r.NewAuthContext()
	if err != nil {
		return nil, op.error(err)
	}
	req := &proto.DirDeleteRequest{
		Name: string(pathName),
	}
	resp, err := r.dirClient.Delete(gCtx, req, callOpt)
	err = finishAuth(err)
	r.checkGrpc(err)
	return op.entryError(resp, err)
}

// Lookup implements upspin.DirServer.Lookup.
func (r *remote) Lookup(pathName upspin.PathName) (*upspin.DirEntry, error) {
	op := r.opf("Lookup", "%q", pathName)

	gCtx, callOpt, finishAuth, err := r.NewAuthContext()
	if err != nil {
		return nil, op.error(err)
	}
	req := &proto.DirLookupRequest{
		Name: string(pathName),
	}
	resp, err := r.dirClient.Lookup(gCtx, req, callOpt)
	err = finishAuth(err)
	r.checkGrpc(err)
	return op.entryError(resp, err)
}

// Watch implements upspin.DirServer.
func (r *remote) Watch(name upspin.PathName, order int64, done <-chan struct{}) (<-chan upspin.Event, error) {
	op := r.opf("Watch", "%q %d", name, order)

	gCtx, callOpt, finishAuth, err := r.NewAuthContext()
	if err != nil {
		return nil, op.error(err)
	}
	srvStream, err := r.dirClient.Watch(gCtx, callOpt)
	err = finishAuth(err)
	if err != nil {
		r.checkGrpc(err)
		return nil, op.error(err)
	}
	req := &proto.DirWatchRequest{
		Name:  string(name),
		Order: order,
	}
	err = srvStream.Send(req)
	if err != nil {
		return nil, op.error(err)
	}

	// First event back determines if Watch call was successful.
	protoEvent, err := srvStream.Recv()
	if err != nil {
		return nil, op.error(err)
	}
	event, err := proto.UpspinEvent(protoEvent)
	if err != nil {
		return nil, op.error(err)
	}
	if event.Error != nil {
		return nil, event.Error
	}
	// Assert event.Entry is nil, since this is a synthetic event, which
	// serves to confirm whether Watch succeeded or not.
	if event.Entry != nil {
		return nil, op.error(errors.Internal, errors.Str("first event must have nil entry"))
	}
	// No error on Watch. Establish output channel.
	events := make(chan upspin.Event, 1)
	serverDone := make(chan struct{})
	go r.watchDone(op, srvStream, events, done, serverDone)
	go r.watch(op, srvStream, events, serverDone)
	return events, nil
}

// watchDone, which runs in a goroutine, waits for either the client to close
// its done channel or the server to close its Events channel. When either one
// happens it closes the send-side GRPC stream, which in turn signals the server
// to finish up on its side.
func (r *remote) watchDone(op *operation, srvStream proto.Dir_WatchClient, events chan upspin.Event, done, serverDone <-chan struct{}) {
	select {
	case <-done:
		op.logf("Client closed done channel")
	case <-serverDone:
		op.logf("Server closed events channel")
	}
	srvStream.CloseSend()
}

// watch, which runs in a goroutine, receives events from the server and relays
// them to the client onto the events channel. When the server terminates the
// stream (sends the error io.EOF), we close the events channel and notify the
// other goroutine that we've terminated.
//
// Note: there's no need for the sending of events to time out since this is
// the client side. It can hang forever. But it probably won't because all other
// layers have timeouts.
func (r *remote) watch(op *operation, srvStream proto.Dir_WatchClient, events chan upspin.Event, serverDone chan struct{}) {
	defer close(events)
	defer close(serverDone) // tell the other goroutine the server is done.

	for {
		protoEvent, err := srvStream.Recv()
		if err == io.EOF {
			// Server closed the channel. Nothing else to do but to
			// close ours too.
			return
		}
		if err != nil {
			op.logErr(err)
			events <- upspin.Event{Error: err}
			return
		}
		event, err := proto.UpspinEvent(protoEvent)
		if err != nil {
			op.logErr(err)
			events <- upspin.Event{Error: err}
			return
		}
		events <- *event
	}
}

// Endpoint implements upspin.StoreServer.Endpoint.
func (r *remote) Endpoint() upspin.Endpoint {
	return r.ctx.endpoint
}

func dialCache(op *operation, context upspin.Context, proxyFor upspin.Endpoint) upspin.Service {
	// Are we using a cache?
	ce := context.CacheEndpoint()
	if ce.Transport == upspin.Unassigned {
		return nil
	}

	// Call the cache. The cache is local so don't bother with TLS.
	authClient, err := grpcauth.NewGRPCClient(context, ce.NetAddr, grpcauth.KeepAliveInterval, grpcauth.NoSecurity, proxyFor)
	if err != nil {
		// On error dial direct.
		op.error(errors.IO, err)
		return nil
	}

	// The connection is closed when this service is released (see Bind.Release).
	dirClient := proto.NewDirClient(authClient.GRPCConn())
	authClient.SetService(dirClient)

	return &remote{
		AuthClientService: authClient,
		ctx: dialContext{
			endpoint: proxyFor,
			userName: context.UserName(),
		},
		dirClient: dirClient,
	}
}

// Dial implements upspin.Service.
func (r *remote) Dial(context upspin.Context, e upspin.Endpoint) (upspin.Service, error) {
	op := r.opf("Dial", "%q, %q", context.UserName(), e)

	if e.Transport != upspin.Remote {
		return nil, op.error(errors.Invalid, errors.Str("unrecognized transport"))
	}

	// First try a cache
	if svc := dialCache(op, context, e); svc != nil {
		return svc, nil
	}

	authClient, err := grpcauth.NewGRPCClient(context, e.NetAddr, grpcauth.KeepAliveInterval, grpcauth.Secure, upspin.Endpoint{})
	if err != nil {
		return nil, op.error(errors.IO, err)
	}

	// The connection is closed when this service is released (see Bind.Release)
	dirClient := proto.NewDirClient(authClient.GRPCConn())
	authClient.SetService(dirClient)
	r = &remote{
		AuthClientService: authClient,
		ctx: dialContext{
			endpoint: e,
			userName: context.UserName(),
		},
		dirClient: dirClient,
	}

	return r, nil
}

const transport = upspin.Remote

func init() {
	r := &remote{} // uninitialized until Dial time.
	bind.RegisterDirServer(transport, r)
}

// unmarshalError calls proto.UnmarshalError, but if the error
// matches upspin.ErrFollowLink, it returns that exact value so
// the caller can do == to test for links.
func unmarshalError(b []byte) error {
	err := errors.UnmarshalError(b)
	if err != nil && err.Error() == upspin.ErrFollowLink.Error() {
		return upspin.ErrFollowLink
	}
	return err
}

func (r *remote) opf(method string, format string, args ...interface{}) *operation {
	ep := r.ctx.endpoint.String()
	s := fmt.Sprintf("dir/remote.%s(%q)", method, ep)
	op := &operation{s, fmt.Sprintf(format, args...)}
	log.Debug.Print(op)
	return op
}

type operation struct {
	op   string
	args string
}

func (op *operation) String() string {
	return fmt.Sprintf("%s(%s)", op.op, op.args)
}

func (op *operation) logErr(err error) {
	log.Error.Printf("%s: %s", op, err)
}

func (op *operation) logf(format string, args ...interface{}) {
	log.Printf("%s: "+format, append([]interface{}{op.op}, args...)...)
}

func (op *operation) error(args ...interface{}) error {
	if len(args) == 0 {
		panic("error called with zero args")
	}
	if len(args) == 1 {
		if e, ok := args[0].(error); ok && e == upspin.ErrFollowLink {
			return e
		}
		if args[0] == nil {
			return nil
		}
	}
	log.Debug.Printf("%v error: %v", op, errors.E(args...))
	return errors.E(append([]interface{}{op.op}, args...)...)
}

// entryError performs the common operation of converting an EntryError
// protocol buffer into a directory entry and error pair.
func (op *operation) entryError(p *proto.EntryError, err error) (*upspin.DirEntry, error) {
	if err != nil {
		return nil, op.error(errors.IO, err)
	}
	err = unmarshalError(p.Error)
	if err != nil && err != upspin.ErrFollowLink {
		return nil, op.error(err)
	}
	entry, unmarshalErr := proto.UpspinDirEntry(p.Entry)
	if unmarshalErr != nil {
		return nil, op.error(unmarshalErr)
	}
	return entry, op.error(err)
}

// GRPC will call back a server if HTTP2 signals that the underlying connection has
// been terminated. However, shiould the server be unreachable, GRPC will poison
// the connection and no longer retry the connection returning a transport error on
// all subsequent RPCs. As a result we will not survive server bounces.
//
// To remedy this, we must recognize this condition and remove the service
// from bind's cache and close the GRPC connection. While grpc has an internal
// ConnectivityState called Shutdown that indicates this condition we have no
// access to it. Instead we must examine the error return from GRPC RPCs.
func (r *remote) checkGrpc(err error) {
	if err == nil || !strings.Contains(err.Error(), "transport: dial") {
		return
	}
	bind.Release(r)
}
<|endoftext|>


<commit_before>package dir_parser

import (
	"fmt"
	"github.com/ibmjstart/cf-download/cmd_exec"
	"github.com/mgutz/ansi"
	"os"
	"regexp"
	"strings"
	"time"
)

type Parser interface {
	ExecParseDir(readPath string) ([]string, []string)
	GetFailedDownloads() []string
	GetDirectory(readPath string) (string, string)
}

type parser struct {
	cmdExec         cmd_exec.CmdExec
	appName         string
	instance        string
	onWindows       bool
	verbose         bool
	failedDownloads []string
}

func NewParser(cmdExec cmd_exec.CmdExec, appName, instance string, onWindows, verbose bool) *parser {
	return &parser{
		cmdExec:   cmdExec,
		appName:   appName,
		instance:  instance,
		onWindows: onWindows,
		verbose:   verbose,
	}
}

/*
*	execParseDir() uses os/exec to shell out commands to cf files with the given readPath. The returned
*	text contains file and directory structure which is then parsed into two slices, dirs and files. dirs
*	contains the names of directories in readPath, files contians the file names. dirs and files are returned
* 	to be downloaded by download() and downloadFile() respectively.
 */
func (p *parser) ExecParseDir(readPath string) ([]string, []string) {
	dir, status := p.GetDirectory(readPath)

	if status == "OK" {
		// parse the returned output into files and dirs slices
		filesSlice := strings.Fields(dir)
		var files, dirs []string
		var name string
		for i := 0; i < len(filesSlice); i++ {
			if strings.HasSuffix(filesSlice[i], "/") {
				name += filesSlice[i]
				dirs = append(dirs, name)
				name = ""
			} else if isDelimiter(filesSlice[i]) {
				if len(name) > 0 {
					name = strings.TrimSuffix(name, " ")
					files = append(files, name)
				}
				name = ""
			} else {
				name += filesSlice[i] + " "
			}
		}
		return files, dirs
	} else {
		//error occured, error message displayed by GetDirectory()
		if len(dir) > 0 {
			fmt.Println(dir)
		}
		if readPath == "/" {
			os.Exit(1)
		}
	}

	return nil, nil
}

/*
*	getDirectory will return the directory as a string ready for parsing.
*	There is a status code returned as well, this is not necessary but helps with testing.
 */
func (p *parser) GetDirectory(readPath string) (string, string) {

	// make the cf files call using exec
	output, err := p.cmdExec.GetFile(p.appName, readPath, p.instance)
	dirSlice := strings.SplitAfterN(string(output), "\n", 3)

	// if cf files fails to get directory, retry (this code is not covered in tests)
	if len(dirSlice) < 2 {
		iterations := 0
		for len(dirSlice) < 2 && iterations < 10 {
			time.Sleep(3 * time.Second)
			output, err = p.cmdExec.GetFile(p.appName, readPath, p.instance)
			dirSlice = strings.SplitAfterN(string(output), "\n", 3)
			iterations++
		}
	}

	if len(dirSlice) >= 2 && strings.Contains(dirSlice[1], "OK") {
		if strings.Compare(dirSlice[2], "\nNo files found\n") == 0 {
			return "", "noFiles"
		}
		return dirSlice[2], "OK"
	} else {
		messsage := createMessage(" Server Error: '"+readPath+"' not downloaded", "yellow", p.onWindows)

		p.failedDownloads = append(p.failedDownloads, messsage)

		if p.verbose {
			fmt.Println(messsage)
		}

		// check for other errors
		if err != nil {
			PrintSlice(dirSlice)
		}
		return string(output), "Failed"
	}
}

func (p *parser) GetFailedDownloads() []string {
	return p.failedDownloads
}

func isDelimiter(str string) bool {
	match, _ := regexp.MatchString("^[0-9]([0-9]|.)*(G|M|B|K)$", str)
	if match == true || str == "-" {
		return true
	}
	return false
}

// prints slices in readable format
func PrintSlice(slice []string) error {
	for index, val := range slice {
		fmt.Println(index, ": ", val)
	}
	return nil
}

func createMessage(message, color string, onWindows bool) string {
	errmsg := ansi.Color(message, color)
	if onWindows == true {
		errmsg = message
	}

	return errmsg
}
<commit_msg>moved error logging inside of --verbose conditional in parser.go<commit_after>package dir_parser

import (
	"fmt"
	"github.com/ibmjstart/cf-download/cmd_exec"
	"github.com/mgutz/ansi"
	"os"
	"regexp"
	"strings"
	"time"
)

type Parser interface {
	ExecParseDir(readPath string) ([]string, []string)
	GetFailedDownloads() []string
	GetDirectory(readPath string) (string, string)
}

type parser struct {
	cmdExec         cmd_exec.CmdExec
	appName         string
	instance        string
	onWindows       bool
	verbose         bool
	failedDownloads []string
}

func NewParser(cmdExec cmd_exec.CmdExec, appName, instance string, onWindows, verbose bool) *parser {
	return &parser{
		cmdExec:   cmdExec,
		appName:   appName,
		instance:  instance,
		onWindows: onWindows,
		verbose:   verbose,
	}
}

/*
*	execParseDir() uses os/exec to shell out commands to cf files with the given readPath. The returned
*	text contains file and directory structure which is then parsed into two slices, dirs and files. dirs
*	contains the names of directories in readPath, files contians the file names. dirs and files are returned
* 	to be downloaded by download() and downloadFile() respectively.
 */
func (p *parser) ExecParseDir(readPath string) ([]string, []string) {
	dir, status := p.GetDirectory(readPath)

	if status == "OK" {
		// parse the returned output into files and dirs slices
		filesSlice := strings.Fields(dir)
		var files, dirs []string
		var name string
		for i := 0; i < len(filesSlice); i++ {
			if strings.HasSuffix(filesSlice[i], "/") {
				name += filesSlice[i]
				dirs = append(dirs, name)
				name = ""
			} else if isDelimiter(filesSlice[i]) {
				if len(name) > 0 {
					name = strings.TrimSuffix(name, " ")
					files = append(files, name)
				}
				name = ""
			} else {
				name += filesSlice[i] + " "
			}
		}
		return files, dirs
	} else {
		//error occured, error message displayed by GetDirectory()
		if len(dir) > 0 {
			fmt.Println(dir)
		}
		if readPath == "/" {
			os.Exit(1)
		}
	}

	return nil, nil
}

/*
*	getDirectory will return the directory as a string ready for parsing.
*	There is a status code returned as well, this is not necessary but helps with testing.
 */
func (p *parser) GetDirectory(readPath string) (string, string) {

	// make the cf files call using exec
	output, err := p.cmdExec.GetFile(p.appName, readPath, p.instance)
	dirSlice := strings.SplitAfterN(string(output), "\n", 3)

	// if cf files fails to get directory, retry (this code is not covered in tests)
	if len(dirSlice) < 2 {
		iterations := 0
		for len(dirSlice) < 2 && iterations < 10 {
			time.Sleep(3 * time.Second)
			output, err = p.cmdExec.GetFile(p.appName, readPath, p.instance)
			dirSlice = strings.SplitAfterN(string(output), "\n", 3)
			iterations++
		}
	}

	if len(dirSlice) >= 2 && strings.Contains(dirSlice[1], "OK") {
		if strings.Compare(dirSlice[2], "\nNo files found\n") == 0 {
			return "", "noFiles"
		}
		return dirSlice[2], "OK"
	} else {
		message := createMessage(" Server Error: '"+readPath+"' not downloaded", "yellow", p.onWindows)

		p.failedDownloads = append(p.failedDownloads, message)

		if p.verbose {
			fmt.Println(message)
			// check for other errors
			if err != nil {
				PrintSlice(dirSlice)
			}
		}

		return string(output), "Failed"
	}
}

func (p *parser) GetFailedDownloads() []string {
	return p.failedDownloads
}

func isDelimiter(str string) bool {
	match, _ := regexp.MatchString("^[0-9]([0-9]|.)*(G|M|B|K)$", str)
	if match == true || str == "-" {
		return true
	}
	return false
}

// prints slices in readable format
func PrintSlice(slice []string) error {
	for index, val := range slice {
		fmt.Println(index, ": ", val)
	}
	return nil
}

func createMessage(message, color string, onWindows bool) string {
	errmsg := ansi.Color(message, color)
	if onWindows == true {
		errmsg = message
	}

	return errmsg
}
<|endoftext|>


<commit_before>package persistlog

import (
	"crypto/sha256"
	"github.com/lightningnetwork/lnd/chainntnfs"
	"github.com/roasbeef/btcd/btcec"
	"github.com/roasbeef/btcd/chaincfg/chainhash"
	"github.com/roasbeef/btcd/wire"
	"math"
	"os"
	"testing"
	"time"
)

var (
	// Bytes of a private key
	key = [32]byte{
		0x81, 0xb6, 0x37, 0xd8, 0xfc, 0xd2, 0xc6, 0xda,
		0x68, 0x59, 0xe6, 0x96, 0x31, 0x13, 0xa1, 0x17,
		0xd, 0xe7, 0x93, 0xe4, 0xb7, 0x25, 0xb8, 0x4d,
		0x1e, 0xb, 0x4c, 0xf9, 0x9e, 0xc5, 0x8c, 0xe9,
	}
)

type mockNotifier struct {
	confChannel chan *chainntnfs.TxConfirmation
	epochChan   chan *chainntnfs.BlockEpoch
}

func (m *mockNotifier) RegisterBlockEpochNtfn() (*chainntnfs.BlockEpochEvent, error) {
	return &chainntnfs.BlockEpochEvent{
		Epochs: m.epochChan,
		Cancel: func() {},
	}, nil
}

func (m *mockNotifier) RegisterConfirmationsNtfn(txid *chainhash.Hash, numConfs,
	heightHint uint32) (*chainntnfs.ConfirmationEvent, error) {
	return nil, nil
}

func (m *mockNotifier) RegisterSpendNtfn(outpoint *wire.OutPoint,
	heightHint uint32) (*chainntnfs.SpendEvent, error) {
	return nil, nil
}

func (m *mockNotifier) Start() error {
	return nil
}

func (m *mockNotifier) Stop() error {
	return nil
}

// generateSharedSecret generates a shared secret given a public key and a
// private key. It is directly copied from sphinx.go.
func generateSharedSecret(pub *btcec.PublicKey, priv *btcec.PrivateKey) [32]byte {
	s := &btcec.PublicKey{}
	x, y := btcec.S256().ScalarMult(pub.X, pub.Y, priv.D.Bytes())
	s.X = x
	s.Y = y

	return sha256.Sum256(s.SerializeCompressed())
}

// shutdown stops the DecayedLog and deletes the folder enclosing the
// temporary channel database.
func shutdown(d *DecayedLog) {
	os.RemoveAll("tempdir")
	d.Stop()
}

// TestDecayedLogGarbageCollector tests the ability of the garbage collector
// to delete expired cltv values every time a block is received. Expired cltv
// values are cltv values that are <= current block height.
func TestDecayedLogGarbageCollector(t *testing.T) {
	// Random (TO-BE-EXPIRED) cltv value
	cltv := uint32(2)

	// Create the MockNotifier which triggers the garbage collector
	MockNotifier := &mockNotifier{
		epochChan: make(chan *chainntnfs.BlockEpoch, 1),
	}

	// Create a DecayedLog object
	d := DecayedLog{Notifier: MockNotifier}

	// Open the channeldb (start the garbage collector)
	err := d.Start("tempdir")
	if err != nil {
		t.Fatalf("Unable to start / open DecayedLog")
	}
	defer shutdown(&d)

	// Create a new private key on elliptic curve secp256k1
	priv, err := btcec.NewPrivateKey(btcec.S256())
	if err != nil {
		t.Fatalf("Unable to create new private key")
	}

	// Generate a public key from the key bytes
	_, testPub := btcec.PrivKeyFromBytes(btcec.S256(), key[:])

	// Generate a shared secret with the public and private keys we made
	secret := generateSharedSecret(testPub, priv)

	// Create the hashedSecret given the shared secret we just generated.
	// This is the first 20 bytes of the Sha-256 hash of the shared secret.
	// This is used as a key to retrieve the cltv value.
	hashedSecret := HashSharedSecret(secret)

	// Store <hashedSecret, cltv> in the sharedHashBucket.
	err = d.Put(hashedSecret[:], cltv)
	if err != nil {
		t.Fatalf("Unable to store in channeldb")
	}

	// Send Block notification to garbage collector. The garbage collector
	// should remove the entry we just added to sharedHashBucket as it is
	// now expired.
	MockNotifier.epochChan <- &chainntnfs.BlockEpoch{
		Height: int32(101),
	}
	MockNotifier.epochChan <- &chainntnfs.BlockEpoch{
		Height: int32(102),
	}

	// Wait for database write (GC is in a goroutine)
	time.Sleep(500 * time.Millisecond)

	// Assert that hashedSecret is not in the sharedHashBucket
	val, err := d.Get(hashedSecret[:])
	if err != nil {
		t.Fatalf("Delete failed - received an error upon Get")
	}

	if val != math.MaxUint32 {
		t.Fatalf("cltv was not deleted")
	}
}

// TestDecayedLogInsertionAndRetrieval inserts a cltv value into the nested
// sharedHashBucket and then deletes it and finally asserts that we can no
// longer retrieve it.
func TestDecayedLogInsertionAndDeletion(t *testing.T) {
	// Random cltv value
	cltv := uint32(5)

	// Create a DecayedLog object
	d := DecayedLog{}

	// Open the channeldb
	err := d.Start("tempdir")
	if err != nil {
		t.Fatalf("Unable to start / open DecayedLog")
	}
	defer shutdown(&d)

	// Create a new private key on elliptic curve secp256k1
	priv, err := btcec.NewPrivateKey(btcec.S256())
	if err != nil {
		t.Fatalf("Unable to create new private key")
	}

	// Generate a public key from the key bytes
	_, testPub := btcec.PrivKeyFromBytes(btcec.S256(), key[:])

	// Generate a shared secret with the public and private keys we made
	secret := generateSharedSecret(testPub, priv)

	// Create the hashedSecret given the shared secret we just generated.
	// This is the first 20 bytes of the Sha-256 hash of the shared secret.
	// This is used as a key to retrieve the cltv value.
	hashedSecret := HashSharedSecret(secret)

	// Store <hashedSecret, cltv> in the sharedHashBucket.
	err = d.Put(hashedSecret[:], cltv)
	if err != nil {
		t.Fatalf("Unable to store in channeldb")
	}

	// Delete hashedSecret from the sharedHashBucket.
	err = d.Delete(hashedSecret[:])
	if err != nil {
		t.Fatalf("Unable to delete from channeldb")
	}

	// Assert that hashedSecret is not in the sharedHashBucket
	val, err := d.Get(hashedSecret[:])
	if err != nil {
		t.Fatalf("Delete failed - received the wrong error message")
	}

	if val != math.MaxUint32 {
		t.Fatalf("cltv was not deleted")
	}

}

// TestDecayedLogStartAndStop tests for persistence. The DecayedLog is started,
// a cltv value is stored in the sharedHashBucket, and then it the DecayedLog
// is stopped. The DecayedLog is then started up again and we test that the
// cltv value is indeed still stored in the sharedHashBucket. We then delete
// the cltv value and check that it persists upon startup.
func TestDecayedLogStartAndStop(t *testing.T) {
	// Random cltv value
	cltv := uint32(6)

	// Create a DecayedLog object
	d := DecayedLog{}

	// Open the channeldb
	err := d.Start("tempdir")
	if err != nil {
		t.Fatalf("Unable to start / open DecayedLog")
	}
	defer shutdown(&d)

	// Create a new private key on elliptic curve secp256k1
	priv, err := btcec.NewPrivateKey(btcec.S256())
	if err != nil {
		t.Fatalf("Unable to create new private key")
	}

	// Generate a public key from the key bytes
	_, testPub := btcec.PrivKeyFromBytes(btcec.S256(), key[:])

	// Generate a shared secret with the public and private keys we made
	secret := generateSharedSecret(testPub, priv)

	// Create the hashedSecret given the shared secret we just generated.
	// This is the first 20 bytes of the Sha-256 hash of the shared secret.
	// This is used as a key to retrieve the cltv value.
	hashedSecret := HashSharedSecret(secret)

	// Store <hashedSecret, cltv> in the sharedHashBucket.
	err = d.Put(hashedSecret[:], cltv)
	if err != nil {
		t.Fatalf("Unable to store in channeldb")
	}

	// Shutdown the DecayedLog's channeldb
	d.Stop()

	// Startup the DecayedLog's channeldb
	err = d.Start("tempdir")
	if err != nil {
		t.Fatalf("Unable to start / open DecayedLog")
	}

	// Retrieve the stored cltv value given the hashedSecret key.
	value, err := d.Get(hashedSecret[:])
	if err != nil {
		t.Fatalf("Unable to retrieve from channeldb")
	}

	// Check that the original cltv value matches the retrieved cltv
	// value.
	if cltv != value {
		t.Fatalf("Value retrieved doesn't match value stored")
	}

	// Delete hashedSecret from sharedHashBucket
	err = d.Delete(hashedSecret[:])
	if err != nil {
		t.Fatalf("Unable to delete from channeldb")
	}

	// Shutdown the DecayedLog's channeldb
	d.Stop()

	// Startup the DecayedLog's channeldb
	err = d.Start("tempdir")
	if err != nil {
		t.Fatalf("Unable to start / open DecayedLog")
	}

	// Assert that hashedSecret is not in the sharedHashBucket
	val, err := d.Get(hashedSecret[:])
	if err != nil {
		t.Fatalf("Delete failed")
	}

	if val != math.MaxUint32 {
		t.Fatalf("cltv was not deleted")
	}

}

// TestDecayedLogStorageAndRetrieval stores a cltv value and then retrieves it
// via the nested sharedHashBucket and finally asserts that the original stored
// and retrieved cltv values are equal.
func TestDecayedLogStorageAndRetrieval(t *testing.T) {
	// Random cltv value
	cltv := uint32(7)

	// Create a DecayedLog object
	d := DecayedLog{}

	// Open the channeldb
	err := d.Start("tempdir")
	if err != nil {
		t.Fatalf("Unable to start / open DecayedLog")
	}
	defer shutdown(&d)

	// Create a new private key on elliptic curve secp256k1
	priv, err := btcec.NewPrivateKey(btcec.S256())
	if err != nil {
		t.Fatalf("Unable to create new private key")
	}

	// Generate a public key from the key bytes
	_, testPub := btcec.PrivKeyFromBytes(btcec.S256(), key[:])

	// Generate a shared secret with the public and private keys we made
	secret := generateSharedSecret(testPub, priv)

	// Create the hashedSecret given the shared secret we just generated.
	// This is the first 20 bytes of the Sha-256 hash of the shared secret.
	// This is used as a key to retrieve the cltv value.
	hashedSecret := HashSharedSecret(secret)

	// Store <hashedSecret, cltv> in the sharedHashBucket
	err = d.Put(hashedSecret[:], cltv)
	if err != nil {
		t.Fatalf("Unable to store in channeldb")
	}

	// Retrieve the stored cltv value given the hashedSecret key.
	value, err := d.Get(hashedSecret[:])
	if err != nil {
		t.Fatalf("Unable to retrieve from channeldb")
	}

	// If the original cltv value does not match the value retrieved,
	// then the test failed.
	if cltv != value {
		t.Fatalf("Value retrieved doesn't match value stored")
	}

}
<commit_msg>small test fix<commit_after>package persistlog

import (
	"crypto/sha256"
	"github.com/lightningnetwork/lnd/chainntnfs"
	"github.com/roasbeef/btcd/btcec"
	"github.com/roasbeef/btcd/chaincfg/chainhash"
	"github.com/roasbeef/btcd/wire"
	"math"
	"os"
	"testing"
	"time"
)

var (
	// Bytes of a private key
	key = [32]byte{
		0x81, 0xb6, 0x37, 0xd8, 0xfc, 0xd2, 0xc6, 0xda,
		0x68, 0x59, 0xe6, 0x96, 0x31, 0x13, 0xa1, 0x17,
		0xd, 0xe7, 0x93, 0xe4, 0xb7, 0x25, 0xb8, 0x4d,
		0x1e, 0xb, 0x4c, 0xf9, 0x9e, 0xc5, 0x8c, 0xe9,
	}
)

type mockNotifier struct {
	confChannel chan *chainntnfs.TxConfirmation
	epochChan   chan *chainntnfs.BlockEpoch
}

func (m *mockNotifier) RegisterBlockEpochNtfn() (*chainntnfs.BlockEpochEvent, error) {
	return &chainntnfs.BlockEpochEvent{
		Epochs: m.epochChan,
		Cancel: func() {},
	}, nil
}

func (m *mockNotifier) RegisterConfirmationsNtfn(txid *chainhash.Hash, numConfs,
	heightHint uint32) (*chainntnfs.ConfirmationEvent, error) {
	return nil, nil
}

func (m *mockNotifier) RegisterSpendNtfn(outpoint *wire.OutPoint,
	heightHint uint32) (*chainntnfs.SpendEvent, error) {
	return nil, nil
}

func (m *mockNotifier) Start() error {
	return nil
}

func (m *mockNotifier) Stop() error {
	return nil
}

// generateSharedSecret generates a shared secret given a public key and a
// private key. It is directly copied from sphinx.go.
func generateSharedSecret(pub *btcec.PublicKey, priv *btcec.PrivateKey) [32]byte {
	s := &btcec.PublicKey{}
	x, y := btcec.S256().ScalarMult(pub.X, pub.Y, priv.D.Bytes())
	s.X = x
	s.Y = y

	return sha256.Sum256(s.SerializeCompressed())
}

// shutdown stops the DecayedLog and deletes the folder enclosing the
// temporary channel database.
func shutdown(d *DecayedLog) {
	os.RemoveAll("tempdir")
	d.Stop()
}

// TestDecayedLogGarbageCollector tests the ability of the garbage collector
// to delete expired cltv values every time a block is received. Expired cltv
// values are cltv values that are <= current block height.
func TestDecayedLogGarbageCollector(t *testing.T) {
	// Random (TO-BE-EXPIRED) cltv value
	cltv := uint32(2)

	// Create the MockNotifier which triggers the garbage collector
	MockNotifier := &mockNotifier{
		epochChan: make(chan *chainntnfs.BlockEpoch, 1),
	}

	// Create a DecayedLog object
	d := DecayedLog{Notifier: MockNotifier}

	// Open the channeldb (start the garbage collector)
	err := d.Start("tempdir")
	if err != nil {
		t.Fatalf("Unable to start / open DecayedLog")
	}
	defer shutdown(&d)

	// Create a new private key on elliptic curve secp256k1
	priv, err := btcec.NewPrivateKey(btcec.S256())
	if err != nil {
		t.Fatalf("Unable to create new private key")
	}

	// Generate a public key from the key bytes
	_, testPub := btcec.PrivKeyFromBytes(btcec.S256(), key[:])

	// Generate a shared secret with the public and private keys we made
	secret := generateSharedSecret(testPub, priv)

	// Create the hashedSecret given the shared secret we just generated.
	// This is the first 20 bytes of the Sha-256 hash of the shared secret.
	// This is used as a key to retrieve the cltv value.
	hashedSecret := HashSharedSecret(secret)

	// Store <hashedSecret, cltv> in the sharedHashBucket.
	err = d.Put(hashedSecret[:], cltv)
	if err != nil {
		t.Fatalf("Unable to store in channeldb")
	}

	// Send Block notification to garbage collector. The garbage collector
	// should remove the entry we just added to sharedHashBucket as it is
	// now expired.
	MockNotifier.epochChan <- &chainntnfs.BlockEpoch{
		Height: int32(101),
	}
	MockNotifier.epochChan <- &chainntnfs.BlockEpoch{
		Height: int32(102),
	}
	MockNotifier.epochChan <- &chainntnfs.BlockEpoch{
		Height: int32(103),
	}

	// Wait for database write (GC is in a goroutine)
	time.Sleep(500 * time.Millisecond)

	// Assert that hashedSecret is not in the sharedHashBucket
	val, err := d.Get(hashedSecret[:])
	if err != nil {
		t.Fatalf("Delete failed - received an error upon Get")
	}

	if val != math.MaxUint32 {
		t.Fatalf("cltv was not deleted")
	}
}

// TestDecayedLogInsertionAndRetrieval inserts a cltv value into the nested
// sharedHashBucket and then deletes it and finally asserts that we can no
// longer retrieve it.
func TestDecayedLogInsertionAndDeletion(t *testing.T) {
	// Random cltv value
	cltv := uint32(5)

	// Create a DecayedLog object
	d := DecayedLog{}

	// Open the channeldb
	err := d.Start("tempdir")
	if err != nil {
		t.Fatalf("Unable to start / open DecayedLog")
	}
	defer shutdown(&d)

	// Create a new private key on elliptic curve secp256k1
	priv, err := btcec.NewPrivateKey(btcec.S256())
	if err != nil {
		t.Fatalf("Unable to create new private key")
	}

	// Generate a public key from the key bytes
	_, testPub := btcec.PrivKeyFromBytes(btcec.S256(), key[:])

	// Generate a shared secret with the public and private keys we made
	secret := generateSharedSecret(testPub, priv)

	// Create the hashedSecret given the shared secret we just generated.
	// This is the first 20 bytes of the Sha-256 hash of the shared secret.
	// This is used as a key to retrieve the cltv value.
	hashedSecret := HashSharedSecret(secret)

	// Store <hashedSecret, cltv> in the sharedHashBucket.
	err = d.Put(hashedSecret[:], cltv)
	if err != nil {
		t.Fatalf("Unable to store in channeldb")
	}

	// Delete hashedSecret from the sharedHashBucket.
	err = d.Delete(hashedSecret[:])
	if err != nil {
		t.Fatalf("Unable to delete from channeldb")
	}

	// Assert that hashedSecret is not in the sharedHashBucket
	val, err := d.Get(hashedSecret[:])
	if err != nil {
		t.Fatalf("Delete failed - received the wrong error message")
	}

	if val != math.MaxUint32 {
		t.Fatalf("cltv was not deleted")
	}

}

// TestDecayedLogStartAndStop tests for persistence. The DecayedLog is started,
// a cltv value is stored in the sharedHashBucket, and then it the DecayedLog
// is stopped. The DecayedLog is then started up again and we test that the
// cltv value is indeed still stored in the sharedHashBucket. We then delete
// the cltv value and check that it persists upon startup.
func TestDecayedLogStartAndStop(t *testing.T) {
	// Random cltv value
	cltv := uint32(6)

	// Create a DecayedLog object
	d := DecayedLog{}

	// Open the channeldb
	err := d.Start("tempdir")
	if err != nil {
		t.Fatalf("Unable to start / open DecayedLog")
	}
	defer shutdown(&d)

	// Create a new private key on elliptic curve secp256k1
	priv, err := btcec.NewPrivateKey(btcec.S256())
	if err != nil {
		t.Fatalf("Unable to create new private key")
	}

	// Generate a public key from the key bytes
	_, testPub := btcec.PrivKeyFromBytes(btcec.S256(), key[:])

	// Generate a shared secret with the public and private keys we made
	secret := generateSharedSecret(testPub, priv)

	// Create the hashedSecret given the shared secret we just generated.
	// This is the first 20 bytes of the Sha-256 hash of the shared secret.
	// This is used as a key to retrieve the cltv value.
	hashedSecret := HashSharedSecret(secret)

	// Store <hashedSecret, cltv> in the sharedHashBucket.
	err = d.Put(hashedSecret[:], cltv)
	if err != nil {
		t.Fatalf("Unable to store in channeldb")
	}

	// Shutdown the DecayedLog's channeldb
	d.Stop()

	// Startup the DecayedLog's channeldb
	err = d.Start("tempdir")
	if err != nil {
		t.Fatalf("Unable to start / open DecayedLog")
	}

	// Retrieve the stored cltv value given the hashedSecret key.
	value, err := d.Get(hashedSecret[:])
	if err != nil {
		t.Fatalf("Unable to retrieve from channeldb")
	}

	// Check that the original cltv value matches the retrieved cltv
	// value.
	if cltv != value {
		t.Fatalf("Value retrieved doesn't match value stored")
	}

	// Delete hashedSecret from sharedHashBucket
	err = d.Delete(hashedSecret[:])
	if err != nil {
		t.Fatalf("Unable to delete from channeldb")
	}

	// Shutdown the DecayedLog's channeldb
	d.Stop()

	// Startup the DecayedLog's channeldb
	err = d.Start("tempdir")
	if err != nil {
		t.Fatalf("Unable to start / open DecayedLog")
	}

	// Assert that hashedSecret is not in the sharedHashBucket
	val, err := d.Get(hashedSecret[:])
	if err != nil {
		t.Fatalf("Delete failed")
	}

	if val != math.MaxUint32 {
		t.Fatalf("cltv was not deleted")
	}

}

// TestDecayedLogStorageAndRetrieval stores a cltv value and then retrieves it
// via the nested sharedHashBucket and finally asserts that the original stored
// and retrieved cltv values are equal.
func TestDecayedLogStorageAndRetrieval(t *testing.T) {
	// Random cltv value
	cltv := uint32(7)

	// Create a DecayedLog object
	d := DecayedLog{}

	// Open the channeldb
	err := d.Start("tempdir")
	if err != nil {
		t.Fatalf("Unable to start / open DecayedLog")
	}
	defer shutdown(&d)

	// Create a new private key on elliptic curve secp256k1
	priv, err := btcec.NewPrivateKey(btcec.S256())
	if err != nil {
		t.Fatalf("Unable to create new private key")
	}

	// Generate a public key from the key bytes
	_, testPub := btcec.PrivKeyFromBytes(btcec.S256(), key[:])

	// Generate a shared secret with the public and private keys we made
	secret := generateSharedSecret(testPub, priv)

	// Create the hashedSecret given the shared secret we just generated.
	// This is the first 20 bytes of the Sha-256 hash of the shared secret.
	// This is used as a key to retrieve the cltv value.
	hashedSecret := HashSharedSecret(secret)

	// Store <hashedSecret, cltv> in the sharedHashBucket
	err = d.Put(hashedSecret[:], cltv)
	if err != nil {
		t.Fatalf("Unable to store in channeldb")
	}

	// Retrieve the stored cltv value given the hashedSecret key.
	value, err := d.Get(hashedSecret[:])
	if err != nil {
		t.Fatalf("Unable to retrieve from channeldb")
	}

	// If the original cltv value does not match the value retrieved,
	// then the test failed.
	if cltv != value {
		t.Fatalf("Value retrieved doesn't match value stored")
	}

}
<|endoftext|>


<commit_before>package hosts

import (
	"bytes"
	"fmt"
	"io"
	"testing"
	"time"

	"github.com/mackerelio/mackerel-client-go"
	"github.com/stretchr/testify/assert"

	"github.com/mackerelio/mkr/mackerelclient"
)

var (
	sampleHost1 = &mackerel.Host{
		ID:          "foo",
		Name:        "sample.app1",
		DisplayName: "Sample Host foo",
		Status:      mackerel.HostStatusWorking,
		Roles: mackerel.Roles{
			"SampleService": []string{"app"},
		},
		IsRetired: false,
		CreatedAt: 1553000000,
		Interfaces: []mackerel.Interface{
			{
				Name:      "en0",
				IPAddress: "10.0.0.1",
			},
		},
	}
	sampleHost2 = &mackerel.Host{
		ID:          "bar",
		Name:        "sample.app2",
		DisplayName: "Sample Host bar",
		Status:      mackerel.HostStatusStandby,
		Roles: mackerel.Roles{
			"SampleService": []string{"db"},
		},
		IsRetired: false,
		CreatedAt: 1552000000,
		Interfaces: []mackerel.Interface{
			{
				Name:      "eth0",
				IPAddress: "10.0.1.2",
			},
		},
	}
)

func TestHostApp_FindHosts(t *testing.T) {
	time.Local = time.FixedZone("Asia/Tokyo", 9*60*60)
	defer func() { time.Local = nil }()
	testCases := []struct {
		id       string
		verbose  bool
		name     string
		service  string
		roles    []string
		statuses []string
		format   string
		hosts    []*mackerel.Host
		expected string
	}{
		{
			id:    "default",
			hosts: []*mackerel.Host{sampleHost1, sampleHost2},
			expected: `[
    {
        "id": "foo",
        "name": "sample.app1",
        "displayName": "Sample Host foo",
        "status": "working",
        "roleFullnames": [
            "SampleService:app"
        ],
        "isRetired": false,
        "createdAt": "2019-03-19T21:53:20+09:00",
        "ipAddresses": {
            "en0": "10.0.0.1"
        }
    },
    {
        "id": "bar",
        "name": "sample.app2",
        "displayName": "Sample Host bar",
        "status": "standby",
        "roleFullnames": [
            "SampleService:db"
        ],
        "isRetired": false,
        "createdAt": "2019-03-08T08:06:40+09:00",
        "ipAddresses": {
            "eth0": "10.0.1.2"
        }
    }
]
`,
		},
		{
			id:      "verbose",
			hosts:   []*mackerel.Host{sampleHost1, sampleHost2},
			verbose: true,
			expected: `[
    {
        "id": "foo",
        "name": "sample.app1",
        "displayName": "Sample Host foo",
        "type": "",
        "status": "working",
        "memo": "",
        "roles": {
            "SampleService": [
                "app"
            ]
        },
        "isRetired": false,
        "createdAt": 1553000000,
        "meta": {},
        "interfaces": [
            {
                "name": "en0",
                "ipAddress": "10.0.0.1"
            }
        ]
    },
    {
        "id": "bar",
        "name": "sample.app2",
        "displayName": "Sample Host bar",
        "type": "",
        "status": "standby",
        "memo": "",
        "roles": {
            "SampleService": [
                "db"
            ]
        },
        "isRetired": false,
        "createdAt": 1552000000,
        "meta": {},
        "interfaces": [
            {
                "name": "eth0",
                "ipAddress": "10.0.1.2"
            }
        ]
    }
]
`,
		},
		{
			id:     "format",
			hosts:  []*mackerel.Host{sampleHost1, sampleHost2},
			format: `{{range .}}{{.ID}} {{.Name}} {{.Status}} {{.CreatedAt}}{{"\n"}}{{end}}`,
			expected: `foo sample.app1 working 1553000000
bar sample.app2 standby 1552000000
`,
		},
		{
			id:       "name",
			hosts:    []*mackerel.Host{},
			name:     "Sample.app",
			expected: "[]\n",
		},
		{
			id:       "service",
			hosts:    []*mackerel.Host{},
			service:  "SampleService",
			expected: "[]\n",
		},
		{
			id:       "roles",
			hosts:    []*mackerel.Host{},
			roles:    []string{"role1", "role2"},
			expected: "[]\n",
		},
		{
			id:       "statuses",
			hosts:    []*mackerel.Host{},
			statuses: []string{mackerel.HostStatusPoweroff, mackerel.HostStatusMaintenance},
			expected: "[]\n",
		},
	}
	for _, tc := range testCases {
		client := mackerelclient.NewMockClient(
			mackerelclient.MockFindHosts(func(param *mackerel.FindHostsParam) ([]*mackerel.Host, error) {
				assert.Equal(t, tc.name, param.Name)
				assert.Equal(t, tc.service, param.Service)
				assert.Equal(t, tc.roles, param.Roles)
				assert.Equal(t, tc.statuses, param.Statuses)
				return tc.hosts, nil
			}),
		)
		t.Run(tc.id, func(t *testing.T) {
			out := new(bytes.Buffer)
			app := &hostApp{
				client:    client,
				outStream: out,
			}
			assert.NoError(t, app.findHosts(findHostsParam{
				verbose:  tc.verbose,
				name:     tc.name,
				service:  tc.service,
				roles:    tc.roles,
				statuses: tc.statuses,
				format:   tc.format,
			}))
			assert.Equal(t, tc.expected, out.String())
		})
	}
}

type testLogger struct {
	w io.Writer
}

func (l *testLogger) Log(prefix, message string) {
	fmt.Fprintln(l.w, prefix, message)
}

func (l *testLogger) Error(err error) {
	fmt.Fprintln(l.w, err.Error())
}

func TestHostApp_CreateHost(t *testing.T) {
	testCases := []struct {
		id                string
		name              string
		roleFullnames     []string
		status            string
		customIdentifier  string
		hostID            string
		output            string
		err               error
		createError       error
		updateStatusError error
	}{
		{
			id:               "basic",
			name:             "app.example.com",
			customIdentifier: "app.hosting.example.com",
			roleFullnames:    []string{"foo:bar"},
			hostID:           "xxx",
			output:           "created xxx\n",
		},
		{
			id:            "with status",
			name:          "app.example.com",
			roleFullnames: []string{"foo:bar"},
			status:        "working",
			hostID:        "xxx",
			output:        "created xxx\nupdated xxx working\n",
		},
		{
			id:            "createError",
			name:          "app.example.com",
			roleFullnames: []string{"foo:bar"},
			hostID:        "xxx",
			output:        "http request failed\n",
			createError:   fmt.Errorf("http request failed"),
			err:           fmt.Errorf("http request failed"),
		},
		{
			id:                "updateStatusError",
			name:              "app.example.com",
			roleFullnames:     []string{"foo:bar"},
			status:            "working",
			hostID:            "xxx",
			output:            "created xxx\nhttp request failed\n",
			updateStatusError: fmt.Errorf("http request failed"),
			err:               fmt.Errorf("http request failed"),
		},
	}
	for _, tc := range testCases {
		client := mackerelclient.NewMockClient(
			mackerelclient.MockCreateHost(func(param *mackerel.CreateHostParam) (string, error) {
				assert.Equal(t, tc.name, param.Name)
				assert.Equal(t, tc.roleFullnames, param.RoleFullnames)
				assert.Equal(t, tc.customIdentifier, param.CustomIdentifier)
				return tc.hostID, tc.createError
			}),
			mackerelclient.MockUpdateHostStatus(func(hostID, status string) error {
				assert.Equal(t, tc.hostID, hostID)
				assert.Equal(t, tc.status, status)
				return tc.updateStatusError
			}),
		)
		t.Run(tc.id, func(t *testing.T) {
			out := new(bytes.Buffer)
			app := &hostApp{
				client:    client,
				logger:    &testLogger{out},
				outStream: out,
			}
			assert.Equal(t, tc.err, app.createHost(createHostParam{
				name:             tc.name,
				roleFullnames:    tc.roleFullnames,
				status:           tc.status,
				customIdentifier: tc.customIdentifier,
			}))
			assert.Equal(t, tc.output, out.String())
		})
	}
}
<commit_msg>drop Type field from Host and introduce Size instead<commit_after>package hosts

import (
	"bytes"
	"fmt"
	"io"
	"testing"
	"time"

	"github.com/mackerelio/mackerel-client-go"
	"github.com/stretchr/testify/assert"

	"github.com/mackerelio/mkr/mackerelclient"
)

var (
	sampleHost1 = &mackerel.Host{
		ID:          "foo",
		Name:        "sample.app1",
		DisplayName: "Sample Host foo",
		Status:      mackerel.HostStatusWorking,
		Roles: mackerel.Roles{
			"SampleService": []string{"app"},
		},
		IsRetired: false,
		CreatedAt: 1553000000,
		Interfaces: []mackerel.Interface{
			{
				Name:      "en0",
				IPAddress: "10.0.0.1",
			},
		},
	}
	sampleHost2 = &mackerel.Host{
		ID:          "bar",
		Name:        "sample.app2",
		DisplayName: "Sample Host bar",
		Status:      mackerel.HostStatusStandby,
		Roles: mackerel.Roles{
			"SampleService": []string{"db"},
		},
		IsRetired: false,
		CreatedAt: 1552000000,
		Interfaces: []mackerel.Interface{
			{
				Name:      "eth0",
				IPAddress: "10.0.1.2",
			},
		},
	}
)

func TestHostApp_FindHosts(t *testing.T) {
	time.Local = time.FixedZone("Asia/Tokyo", 9*60*60)
	defer func() { time.Local = nil }()
	testCases := []struct {
		id       string
		verbose  bool
		name     string
		service  string
		roles    []string
		statuses []string
		format   string
		hosts    []*mackerel.Host
		expected string
	}{
		{
			id:    "default",
			hosts: []*mackerel.Host{sampleHost1, sampleHost2},
			expected: `[
    {
        "id": "foo",
        "name": "sample.app1",
        "displayName": "Sample Host foo",
        "status": "working",
        "roleFullnames": [
            "SampleService:app"
        ],
        "isRetired": false,
        "createdAt": "2019-03-19T21:53:20+09:00",
        "ipAddresses": {
            "en0": "10.0.0.1"
        }
    },
    {
        "id": "bar",
        "name": "sample.app2",
        "displayName": "Sample Host bar",
        "status": "standby",
        "roleFullnames": [
            "SampleService:db"
        ],
        "isRetired": false,
        "createdAt": "2019-03-08T08:06:40+09:00",
        "ipAddresses": {
            "eth0": "10.0.1.2"
        }
    }
]
`,
		},
		{
			id:      "verbose",
			hosts:   []*mackerel.Host{sampleHost1, sampleHost2},
			verbose: true,
			expected: `[
    {
        "id": "foo",
        "name": "sample.app1",
        "displayName": "Sample Host foo",
        "size": "",
        "status": "working",
        "memo": "",
        "roles": {
            "SampleService": [
                "app"
            ]
        },
        "isRetired": false,
        "createdAt": 1553000000,
        "meta": {},
        "interfaces": [
            {
                "name": "en0",
                "ipAddress": "10.0.0.1"
            }
        ]
    },
    {
        "id": "bar",
        "name": "sample.app2",
        "displayName": "Sample Host bar",
        "size": "",
        "status": "standby",
        "memo": "",
        "roles": {
            "SampleService": [
                "db"
            ]
        },
        "isRetired": false,
        "createdAt": 1552000000,
        "meta": {},
        "interfaces": [
            {
                "name": "eth0",
                "ipAddress": "10.0.1.2"
            }
        ]
    }
]
`,
		},
		{
			id:     "format",
			hosts:  []*mackerel.Host{sampleHost1, sampleHost2},
			format: `{{range .}}{{.ID}} {{.Name}} {{.Status}} {{.CreatedAt}}{{"\n"}}{{end}}`,
			expected: `foo sample.app1 working 1553000000
bar sample.app2 standby 1552000000
`,
		},
		{
			id:       "name",
			hosts:    []*mackerel.Host{},
			name:     "Sample.app",
			expected: "[]\n",
		},
		{
			id:       "service",
			hosts:    []*mackerel.Host{},
			service:  "SampleService",
			expected: "[]\n",
		},
		{
			id:       "roles",
			hosts:    []*mackerel.Host{},
			roles:    []string{"role1", "role2"},
			expected: "[]\n",
		},
		{
			id:       "statuses",
			hosts:    []*mackerel.Host{},
			statuses: []string{mackerel.HostStatusPoweroff, mackerel.HostStatusMaintenance},
			expected: "[]\n",
		},
	}
	for _, tc := range testCases {
		client := mackerelclient.NewMockClient(
			mackerelclient.MockFindHosts(func(param *mackerel.FindHostsParam) ([]*mackerel.Host, error) {
				assert.Equal(t, tc.name, param.Name)
				assert.Equal(t, tc.service, param.Service)
				assert.Equal(t, tc.roles, param.Roles)
				assert.Equal(t, tc.statuses, param.Statuses)
				return tc.hosts, nil
			}),
		)
		t.Run(tc.id, func(t *testing.T) {
			out := new(bytes.Buffer)
			app := &hostApp{
				client:    client,
				outStream: out,
			}
			assert.NoError(t, app.findHosts(findHostsParam{
				verbose:  tc.verbose,
				name:     tc.name,
				service:  tc.service,
				roles:    tc.roles,
				statuses: tc.statuses,
				format:   tc.format,
			}))
			assert.Equal(t, tc.expected, out.String())
		})
	}
}

type testLogger struct {
	w io.Writer
}

func (l *testLogger) Log(prefix, message string) {
	fmt.Fprintln(l.w, prefix, message)
}

func (l *testLogger) Error(err error) {
	fmt.Fprintln(l.w, err.Error())
}

func TestHostApp_CreateHost(t *testing.T) {
	testCases := []struct {
		id                string
		name              string
		roleFullnames     []string
		status            string
		customIdentifier  string
		hostID            string
		output            string
		err               error
		createError       error
		updateStatusError error
	}{
		{
			id:               "basic",
			name:             "app.example.com",
			customIdentifier: "app.hosting.example.com",
			roleFullnames:    []string{"foo:bar"},
			hostID:           "xxx",
			output:           "created xxx\n",
		},
		{
			id:            "with status",
			name:          "app.example.com",
			roleFullnames: []string{"foo:bar"},
			status:        "working",
			hostID:        "xxx",
			output:        "created xxx\nupdated xxx working\n",
		},
		{
			id:            "createError",
			name:          "app.example.com",
			roleFullnames: []string{"foo:bar"},
			hostID:        "xxx",
			output:        "http request failed\n",
			createError:   fmt.Errorf("http request failed"),
			err:           fmt.Errorf("http request failed"),
		},
		{
			id:                "updateStatusError",
			name:              "app.example.com",
			roleFullnames:     []string{"foo:bar"},
			status:            "working",
			hostID:            "xxx",
			output:            "created xxx\nhttp request failed\n",
			updateStatusError: fmt.Errorf("http request failed"),
			err:               fmt.Errorf("http request failed"),
		},
	}
	for _, tc := range testCases {
		client := mackerelclient.NewMockClient(
			mackerelclient.MockCreateHost(func(param *mackerel.CreateHostParam) (string, error) {
				assert.Equal(t, tc.name, param.Name)
				assert.Equal(t, tc.roleFullnames, param.RoleFullnames)
				assert.Equal(t, tc.customIdentifier, param.CustomIdentifier)
				return tc.hostID, tc.createError
			}),
			mackerelclient.MockUpdateHostStatus(func(hostID, status string) error {
				assert.Equal(t, tc.hostID, hostID)
				assert.Equal(t, tc.status, status)
				return tc.updateStatusError
			}),
		)
		t.Run(tc.id, func(t *testing.T) {
			out := new(bytes.Buffer)
			app := &hostApp{
				client:    client,
				logger:    &testLogger{out},
				outStream: out,
			}
			assert.Equal(t, tc.err, app.createHost(createHostParam{
				name:             tc.name,
				roleFullnames:    tc.roleFullnames,
				status:           tc.status,
				customIdentifier: tc.customIdentifier,
			}))
			assert.Equal(t, tc.output, out.String())
		})
	}
}
<|endoftext|>


<commit_before>package input

import "fmt"
import "os"
import "bufio"
import "strings"
import "strconv"

const IntMax = int(^uint(0) >> 1)
const IntMin = -IntMax - 1

// Line reading closure, to save on bufio.NewReader overhead.
// Prompts with a supplied string.
func GetReader() func(string) (string) {
    in := bufio.NewReader(os.Stdin)
    return func(p string) (string) {
        fmt.Print(p)
        s, _ := in.ReadString('\n')
        s = strings.TrimRight(s, "\n\r")
        return s
    }
}

//Type parser func shorthand...
type Parser func(string, interface{}) error

//Parses a line from stdin with specified func,
//retrying indefinitely on failure
func Read(p string,
          f func(string, interface{}) error,
          x interface{}) {
    in := bufio.NewReader(os.Stdin)
    for {
        fmt.Print(p)
        s, e := in.ReadString('\n') //REPLACE _
        s = strings.TrimRight(s, "\n\r")
        if e = f(s, x); e == nil {
            break
        }
        fmt.Println("Not an integer")
    }
}
//Function to parse ints, for use with input.Read()
func Int(s string, x interface{}) error {
    i, e := strconv.Atoi(s)
    if e != nil {
        return e
    }
    *x.(*int) = i
    return nil
}

// Prompts with p until a valid int is entered
func ReadInt(p string) int {
    return ReadRangedInt(p, IntMin, IntMax)
}

func ReadRangedInt(p string, min, max int) int {
    in := GetReader()
    for {
        s := in(p)
        i, e := strconv.Atoi(s)
        if e == nil {
            if i >= min && i <= max {
                return i
            }
            fmt.Printf("Enter a number between %d & %d\n", min, max)
        } else {
            fmt.Println("Not an integer")
        }
    }
}
<commit_msg>Swapping an int-specific error message into input.Int() where it belongs, from input.Read()<commit_after>package input

import "fmt"
import "os"
import "bufio"
import "strings"
import "strconv"

const IntMax = int(^uint(0) >> 1)
const IntMin = -IntMax - 1

// Line reading closure, to save on bufio.NewReader overhead.
// Prompts with a supplied string.
func GetReader() func(string) (string) {
    in := bufio.NewReader(os.Stdin)
    return func(p string) (string) {
        fmt.Print(p)
        s, _ := in.ReadString('\n')
        s = strings.TrimRight(s, "\n\r")
        return s
    }
}

//Type parser func shorthand...
type Parser func(string, interface{}) error

//Parses a line from stdin with specified func,
//retrying indefinitely on failure
func Read(p string,
          f func(string, interface{}) error,
          x interface{}) {
    in := bufio.NewReader(os.Stdin)
    for {
        fmt.Print(p)
        s, e := in.ReadString('\n') //REPLACE _
        s = strings.TrimRight(s, "\n\r")
        if e = f(s, x); e == nil {
            break
        }
        fmt.Println(e)
    }
}
//Function to parse ints, for use with input.Read()
func Int(s string, x interface{}) error {
    i, e := strconv.Atoi(s)
    if e != nil {
        return fmt.Errorf("Not an integer")
    }
    *x.(*int) = i
    return nil
}

// Prompts with p until a valid int is entered
func ReadInt(p string) int {
    return ReadRangedInt(p, IntMin, IntMax)
}

func ReadRangedInt(p string, min, max int) int {
    in := GetReader()
    for {
        s := in(p)
        i, e := strconv.Atoi(s)
        if e == nil {
            if i >= min && i <= max {
                return i
            }
            fmt.Printf("Enter a number between %d & %d\n", min, max)
        } else {
            fmt.Println("Not an integer")
        }
    }
}
<|endoftext|>


<commit_before>package discover

//
//
// TODO:
//   should be able to guess article link format statistically
//   handle/allow subdomains (eg: www1.politicalbetting.com)
//   filter unwanted navlinks (eg "mirror.co.uk/all-about/fred bloggs")
//   HTTP error handling
//   multiple url formats (eg spectator has multiple cms's)
//   logging

import (
	"code.google.com/p/cascadia"
	"code.google.com/p/go.net/html"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"regexp"
	"strings"
)

type Logger interface {
	Printf(format string, v ...interface{})
}

type NullLogger struct{}

func (l NullLogger) Printf(format string, v ...interface{}) {
}

type DiscovererDef struct {
	Name   string
	URL    string
	ArtPat []string
	NavSel string
	// BaseErrorThreshold is starting number of http errors to accept before
	// bailing out.
	// error threshold formula: base + 10% of successful request count
	BaseErrorThreshold int

	// Hostpat is a regex matching accepted domains
	// if empty, reject everything on a different domain
	HostPat string

	// If NoStripQuery is set then article URLs won't have the query part zapped
	NoStripQuery bool
}

type DiscoverStats struct {
	ErrorCount int
	FetchCount int
}

type Discoverer struct {
	Name               string
	StartURL           url.URL
	ArtPats            []*regexp.Regexp
	NavLinkSel         cascadia.Selector
	BaseErrorThreshold int
	StripFragments     bool
	StripQuery         bool
	HostPat            *regexp.Regexp

	ErrorLog Logger
	InfoLog  Logger
	Stats    DiscoverStats
}

func NewDiscoverer(cfg DiscovererDef) (*Discoverer, error) {
	disc := &Discoverer{}
	u, err := url.Parse(cfg.URL)
	if err != nil {
		return nil, err
	}
	disc.Name = cfg.Name
	disc.StartURL = *u
	disc.ArtPats = make([]*regexp.Regexp, 0, len(cfg.ArtPat))
	for _, pat := range cfg.ArtPat {
		re, err := regexp.Compile(pat)
		if err != nil {
			return nil, err
		}
		disc.ArtPats = append(disc.ArtPats, re)
	}

	if cfg.NavSel == "" {
		disc.NavLinkSel = nil
	} else {
		sel, err := cascadia.Compile(cfg.NavSel)
		if err != nil {
			return nil, err
		}
		disc.NavLinkSel = sel
	}
	disc.BaseErrorThreshold = cfg.BaseErrorThreshold

	if cfg.HostPat != "" {
		re, err := regexp.Compile(cfg.HostPat)
		if err != nil {
			return nil, err
		}
		disc.HostPat = re
	}

	// defaults
	disc.StripFragments = true
	disc.StripQuery = !cfg.NoStripQuery
	disc.ErrorLog = NullLogger{}
	disc.InfoLog = NullLogger{}
	return disc, nil
}

func (disc *Discoverer) Run(client *http.Client) (LinkSet, error) {
	// reset stats
	disc.Stats = DiscoverStats{}

	queued := make(LinkSet) // nav pages to scan for article links
	seen := make(LinkSet)   // nav pages we've scanned
	arts := make(LinkSet)   // article links we've found so far

	queued.Add(disc.StartURL)

	for len(queued) > 0 {
		u := queued.Pop()
		seen.Add(u)
		//

		root, err := disc.fetchAndParse(client, &u)
		if err != nil {
			disc.ErrorLog.Printf("%s\n", err.Error())
			disc.Stats.ErrorCount++
			if disc.Stats.ErrorCount > disc.BaseErrorThreshold+(disc.Stats.FetchCount/10) {
				return nil, errors.New("Error threshold exceeded")
			} else {
				continue
			}
		}
		disc.Stats.FetchCount++

		navLinks, err := disc.findNavLinks(root)
		if err != nil {
			return nil, err
		}
		for navLink, _ := range navLinks {
			if _, got := seen[navLink]; !got {
				queued.Add(navLink)
			}
		}

		foo, err := disc.findArticles(root)
		if err != nil {
			return nil, err
		}
		arts.Merge(foo)

		disc.InfoLog.Printf("Visited %s, found %d articles\n", u.String(), len(foo))
	}

	return arts, nil
}

func (disc *Discoverer) fetchAndParse(client *http.Client, pageURL *url.URL) (*html.Node, error) {
	resp, err := client.Get(pageURL.String())
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		err = errors.New(fmt.Sprintf("HTTP code %d (%s)", resp.StatusCode, pageURL.String()))

		return nil, err

	}

	root, err := html.Parse(resp.Body)
	if err != nil {
		return nil, err
	}

	return root, nil
}

var aSel cascadia.Selector = cascadia.MustCompile("a")

func (disc *Discoverer) findArticles(root *html.Node) (LinkSet, error) {
	arts := make(LinkSet)
	for _, a := range aSel.MatchAll(root) {
		// fetch url and extend to absolute
		link, err := disc.StartURL.Parse(GetAttr(a, "href"))
		if err != nil {
			continue
		}

		if !disc.isHostGood(link.Host) {
			continue
		}

		foo := link.RequestURI()
		accept := false
		for _, pat := range disc.ArtPats {
			if pat.MatchString(foo) {
				accept = true
				break
			}
		}
		if !accept {
			continue
		}

		if disc.StripFragments {
			link.Fragment = ""
		}
		if disc.StripQuery {
			link.RawQuery = ""
		}

		arts[*link] = true
	}
	return arts, nil
}

func (disc *Discoverer) findNavLinks(root *html.Node) (LinkSet, error) {
	navLinks := make(LinkSet)
	if disc.NavLinkSel == nil {
		return navLinks, nil
	}
	for _, a := range disc.NavLinkSel.MatchAll(root) {
		link, err := disc.StartURL.Parse(GetAttr(a, "href"))
		if err != nil {
			continue
		}

		if !disc.isHostGood(link.Host) {
			continue
		}

		link.Fragment = ""

		navLinks[*link] = true
	}
	return navLinks, nil
}

// is host domain one we'll accept?
func (disc *Discoverer) isHostGood(host string) bool {
	if disc.HostPat != nil {
		return disc.HostPat.MatchString(host)
	}
	return host == disc.StartURL.Host
}

// GetAttr retrieved the value of an attribute on a node.
// Returns empty string if attribute doesn't exist.
func GetAttr(n *html.Node, attr string) string {
	for _, a := range n.Attr {
		if a.Key == attr {
			return a.Val
		}
	}
	return ""
}

// GetTextContent recursively fetches the text for a node
func GetTextContent(n *html.Node) string {
	if n.Type == html.TextNode {
		return n.Data
	}
	txt := ""
	for child := n.FirstChild; child != nil; child = child.NextSibling {
		txt += GetTextContent(child)
	}

	return txt
}

// CompressSpace reduces all whitespace sequences (space, tabs, newlines etc) in a string to a single space.
// Leading/trailing space is trimmed.
// Has the effect of converting multiline strings to one line.
func CompressSpace(s string) string {
	multispacePat := regexp.MustCompile(`[\s]+`)
	s = strings.TrimSpace(multispacePat.ReplaceAllLiteralString(s, " "))
	return s
}
<commit_msg>use correct base URL for checking article links<commit_after>package discover

//
//
// TODO:
//   should be able to guess article link format statistically
//   handle/allow subdomains (eg: www1.politicalbetting.com)
//   filter unwanted navlinks (eg "mirror.co.uk/all-about/fred bloggs")
//   HTTP error handling
//   multiple url formats (eg spectator has multiple cms's)
//   logging

import (
	"code.google.com/p/cascadia"
	"code.google.com/p/go.net/html"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"regexp"
	"strings"
)

type Logger interface {
	Printf(format string, v ...interface{})
}

type NullLogger struct{}

func (l NullLogger) Printf(format string, v ...interface{}) {
}

type DiscovererDef struct {
	Name   string
	URL    string
	ArtPat []string
	NavSel string
	// BaseErrorThreshold is starting number of http errors to accept before
	// bailing out.
	// error threshold formula: base + 10% of successful request count
	BaseErrorThreshold int

	// Hostpat is a regex matching accepted domains
	// if empty, reject everything on a different domain
	HostPat string

	// If NoStripQuery is set then article URLs won't have the query part zapped
	NoStripQuery bool
}

type DiscoverStats struct {
	ErrorCount int
	FetchCount int
}

type Discoverer struct {
	Name               string
	StartURL           url.URL
	ArtPats            []*regexp.Regexp
	NavLinkSel         cascadia.Selector
	BaseErrorThreshold int
	StripFragments     bool
	StripQuery         bool
	HostPat            *regexp.Regexp

	ErrorLog Logger
	InfoLog  Logger
	Stats    DiscoverStats
}

func NewDiscoverer(cfg DiscovererDef) (*Discoverer, error) {
	disc := &Discoverer{}
	u, err := url.Parse(cfg.URL)
	if err != nil {
		return nil, err
	}
	disc.Name = cfg.Name
	disc.StartURL = *u
	disc.ArtPats = make([]*regexp.Regexp, 0, len(cfg.ArtPat))
	for _, pat := range cfg.ArtPat {
		re, err := regexp.Compile(pat)
		if err != nil {
			return nil, err
		}
		disc.ArtPats = append(disc.ArtPats, re)
	}

	if cfg.NavSel == "" {
		disc.NavLinkSel = nil
	} else {
		sel, err := cascadia.Compile(cfg.NavSel)
		if err != nil {
			return nil, err
		}
		disc.NavLinkSel = sel
	}
	disc.BaseErrorThreshold = cfg.BaseErrorThreshold

	if cfg.HostPat != "" {
		re, err := regexp.Compile(cfg.HostPat)
		if err != nil {
			return nil, err
		}
		disc.HostPat = re
	}

	// defaults
	disc.StripFragments = true
	disc.StripQuery = !cfg.NoStripQuery
	disc.ErrorLog = NullLogger{}
	disc.InfoLog = NullLogger{}
	return disc, nil
}

func (disc *Discoverer) Run(client *http.Client) (LinkSet, error) {
	// reset stats
	disc.Stats = DiscoverStats{}

	queued := make(LinkSet) // nav pages to scan for article links
	seen := make(LinkSet)   // nav pages we've scanned
	arts := make(LinkSet)   // article links we've found so far

	queued.Add(disc.StartURL)

	for len(queued) > 0 {
		pageURL := queued.Pop()
		seen.Add(pageURL)
		//

		root, err := disc.fetchAndParse(client, &pageURL)
		if err != nil {
			disc.ErrorLog.Printf("%s\n", err.Error())
			disc.Stats.ErrorCount++
			if disc.Stats.ErrorCount > disc.BaseErrorThreshold+(disc.Stats.FetchCount/10) {
				return nil, errors.New("Error threshold exceeded")
			} else {
				continue
			}
		}
		disc.Stats.FetchCount++

		navLinks, err := disc.findNavLinks(root)
		if err != nil {
			return nil, err
		}
		for navLink, _ := range navLinks {
			if _, got := seen[navLink]; !got {
				queued.Add(navLink)
			}
		}

		foo, err := disc.findArticles(&pageURL, root)
		if err != nil {
			return nil, err
		}
		arts.Merge(foo)

		disc.InfoLog.Printf("Visited %s, found %d articles\n", pageURL.String(), len(foo))
	}

	return arts, nil
}

func (disc *Discoverer) fetchAndParse(client *http.Client, pageURL *url.URL) (*html.Node, error) {
	resp, err := client.Get(pageURL.String())
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		err = errors.New(fmt.Sprintf("HTTP code %d (%s)", resp.StatusCode, pageURL.String()))

		return nil, err

	}

	root, err := html.Parse(resp.Body)
	if err != nil {
		return nil, err
	}

	return root, nil
}

var aSel cascadia.Selector = cascadia.MustCompile("a")

func (disc *Discoverer) findArticles(baseURL *url.URL, root *html.Node) (LinkSet, error) {
	arts := make(LinkSet)
	for _, a := range aSel.MatchAll(root) {
		u, err := disc.CookArticleURL(baseURL, GetAttr(a, "href"))
		if err != nil {
			continue
		}
		arts[*u] = true
	}
	return arts, nil
}

func (disc *Discoverer) CookArticleURL(baseURL *url.URL, artLink string) (*url.URL, error) {
	// parse, extending to absolute
	u, err := baseURL.Parse(artLink)
	if err != nil {
		return nil, err
	}

	// on a host we accept?
	if !disc.isHostGood(u.Host) {
		return nil, fmt.Errorf("host rejected (%s)", u.Host)
	}

	// matches one of our url forms
	foo := u.RequestURI()
	accept := false
	for _, pat := range disc.ArtPats {
		if pat.MatchString(foo) {
			accept = true
			break
		}
	}
	if !accept {
		return nil, fmt.Errorf("url rejected")
	}

	// apply our sanitising rules for this site
	if disc.StripFragments {
		u.Fragment = ""
	}
	if disc.StripQuery {
		u.RawQuery = ""
	}
	return u, nil
}

func (disc *Discoverer) findNavLinks(root *html.Node) (LinkSet, error) {
	navLinks := make(LinkSet)
	if disc.NavLinkSel == nil {
		return navLinks, nil
	}
	for _, a := range disc.NavLinkSel.MatchAll(root) {
		link, err := disc.StartURL.Parse(GetAttr(a, "href"))
		if err != nil {
			continue
		}

		if !disc.isHostGood(link.Host) {
			continue
		}

		link.Fragment = ""

		navLinks[*link] = true
	}
	return navLinks, nil
}

// is host domain one we'll accept?
func (disc *Discoverer) isHostGood(host string) bool {
	if disc.HostPat != nil {
		return disc.HostPat.MatchString(host)
	}
	return host == disc.StartURL.Host
}

// GetAttr retrieved the value of an attribute on a node.
// Returns empty string if attribute doesn't exist.
func GetAttr(n *html.Node, attr string) string {
	for _, a := range n.Attr {
		if a.Key == attr {
			return a.Val
		}
	}
	return ""
}

// GetTextContent recursively fetches the text for a node
func GetTextContent(n *html.Node) string {
	if n.Type == html.TextNode {
		return n.Data
	}
	txt := ""
	for child := n.FirstChild; child != nil; child = child.NextSibling {
		txt += GetTextContent(child)
	}

	return txt
}

// CompressSpace reduces all whitespace sequences (space, tabs, newlines etc) in a string to a single space.
// Leading/trailing space is trimmed.
// Has the effect of converting multiline strings to one line.
func CompressSpace(s string) string {
	multispacePat := regexp.MustCompile(`[\s]+`)
	s = strings.TrimSpace(multispacePat.ReplaceAllLiteralString(s, " "))
	return s
}
<|endoftext|>


<commit_before>package cmd

import (
	"errors"
	"fmt"
	"os"
	"os/user"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/gobuffalo/envy"
	"github.com/gobuffalo/velvet"
	"github.com/markbates/inflect"
	"github.com/spf13/cobra"
)

var force bool
var verbose bool
var skipPop bool
var skipWebpack bool
var dbType = "postgres"
var ciProvider = "travis"

var newCmd = &cobra.Command{
	Use:   "new [name]",
	Short: "Creates a new Buffalo application",
	RunE: func(cmd *cobra.Command, args []string) error {
		if !validDbType() {
			return fmt.Errorf("Unknown db-type %s expecting one of postgres, mysql or sqlite3", dbType)
		}

		if len(args) == 0 {
			return errors.New("you must enter a name for your new application")
		}

		name := args[0]

		err := validateInGoPath(name)
		if err != nil {
			return err
		}

		rootPath, err := rootPath(name)
		if err != nil {
			return err
		}

		s, _ := os.Stat(rootPath)
		if s != nil {
			if force {
				os.RemoveAll(rootPath)
			} else {
				return fmt.Errorf("%s already exists! Either delete it or use the -f flag to force", name)
			}
		}

		return genNewFiles(name, rootPath)
	},
}

func validDbType() bool {
	return dbType == "postgres" || dbType == "mysql" || dbType == "sqlite3"
}

func validateInGoPath(name string) error {
	gp, err := envy.MustGet("GOPATH")
	if err != nil {
		fmt.Println(noGoPath)
		os.Exit(-1)
	}

	root, err := rootPath(name)
	if err != nil {
		return err
	}

	var gpMultiple []string

	if runtime.GOOS == "windows" {
		gpMultiple = strings.Split(gp, ";") // Windows uses a different separator
	} else {
		gpMultiple = strings.Split(gp, ":")
	}
	gpMultipleLen := len(gpMultiple)
	foundInPath := false

	for i := 0; i < gpMultipleLen; i++ {
		if strings.HasPrefix(root, filepath.Join(gpMultiple[i], "src")) {
			foundInPath = true
			break
		}
	}

	if !foundInPath {
		u, err := user.Current()
		if err != nil {
			return err
		}
		t, err := velvet.Render(notInGoWorkspace, velvet.NewContextWith(map[string]interface{}{
			"name":     name,
			"gopath":   gp,
			"current":  root,
			"username": u.Username,
		}))
		if err != nil {
			return err
		}
		fmt.Println(t)
		os.Exit(-1)
	}
	return nil
}

func goPath(root string) string {
	var gpMultiple []string
	gp := os.Getenv("GOPATH")

	if runtime.GOOS == "windows" {
		gpMultiple = strings.Split(gp, ";") // Windows uses a different separator
	} else {
		gpMultiple = strings.Split(gp, ":")
	}
	gpMultipleLen := len(gpMultiple)
	path := ""

	for i := 0; i < gpMultipleLen; i++ {
		if strings.HasPrefix(root, filepath.Join(gpMultiple[i], "src")) {
			path = gpMultiple[i]
			break
		}
	}
	return path
}

func rootPath(name string) (string, error) {
	pwd, err := os.Getwd()
	if err != nil {
		return "", err
	}
	rootPath := filepath.Join(pwd, name)
	return rootPath, nil
}

func packagePath(rootPath string) string {
	gosrcpath := strings.Replace(filepath.Join(goPath(rootPath), "src"), "\\", "/", -1)
	rootPath = strings.Replace(rootPath, "\\", "/", -1)
	return strings.Replace(rootPath, gosrcpath+"/", "", 2)
}

func genNewFiles(name, rootPath string) error {
	packagePath := packagePath(rootPath)

	data := map[string]interface{}{
		"name":        name,
		"titleName":   inflect.Titleize(name),
		"packagePath": packagePath,
		"actionsPath": packagePath + "/actions",
		"modelsPath":  packagePath + "/models",
		"withPop":     !skipPop,
		"withWebpack": !skipWebpack,
		"dbType":      dbType,
		"version":     Version,
		"ciProvider":  ciProvider,
	}

	g := newAppGenerator(data)
	return g.Run(rootPath, data)
}

func init() {
	RootCmd.AddCommand(newCmd)
	newCmd.Flags().BoolVarP(&force, "force", "f", false, "delete and remake if the app already exists")
	newCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "verbosely print out the go get/install commands")
	newCmd.Flags().BoolVar(&skipPop, "skip-pop", false, "skips adding pop/soda to your app")
	newCmd.Flags().BoolVar(&skipWebpack, "skip-webpack", false, "skips adding Webpack to your app")
	newCmd.Flags().StringVar(&dbType, "db-type", "postgres", "specify the type of database you want to use [postgres, mysql, sqlite3]")
	newCmd.Flags().StringVar(&ciProvider, "ci-provider", "travis", "specify the type of ci file you would like buffalo to generate [travis, none]")
}

const notInGoWorkspace = `Oops! It would appear that you are not in your Go Workspace.

Your $GOPATH is set to "{{gopath}}".

You are currently in "{{current}}".

The standard location for putting Go projects is something along the lines of "$GOPATH/src/github.com/{{username}}/{{name}}" (adjust accordingly).

We recommend you go to "$GOPATH/src/github.com/{{username}}/" and try "buffalo new {{name}}" again.`

const noGoPath = `You do not have a $GOPATH set. In order to work with Go, you must set up your $GOPATH and your Go Workspace.

We recommend reading this tutorial on setting everything up: https://www.goinggo.net/2016/05/installing-go-and-your-workspace.html

When you're ready come back and try again. Don't worry, Buffalo will be right here waiting for you. :)`
<commit_msg>making 'none' the default option for ci provider<commit_after>package cmd

import (
	"errors"
	"fmt"
	"os"
	"os/user"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/gobuffalo/envy"
	"github.com/gobuffalo/velvet"
	"github.com/markbates/inflect"
	"github.com/spf13/cobra"
)

var force bool
var verbose bool
var skipPop bool
var skipWebpack bool
var dbType = "postgres"
var ciProvider = "none"

var newCmd = &cobra.Command{
	Use:   "new [name]",
	Short: "Creates a new Buffalo application",
	RunE: func(cmd *cobra.Command, args []string) error {
		if !validDbType() {
			return fmt.Errorf("Unknown db-type %s expecting one of postgres, mysql or sqlite3", dbType)
		}

		if len(args) == 0 {
			return errors.New("you must enter a name for your new application")
		}

		name := args[0]

		err := validateInGoPath(name)
		if err != nil {
			return err
		}

		rootPath, err := rootPath(name)
		if err != nil {
			return err
		}

		s, _ := os.Stat(rootPath)
		if s != nil {
			if force {
				os.RemoveAll(rootPath)
			} else {
				return fmt.Errorf("%s already exists! Either delete it or use the -f flag to force", name)
			}
		}

		return genNewFiles(name, rootPath)
	},
}

func validDbType() bool {
	return dbType == "postgres" || dbType == "mysql" || dbType == "sqlite3"
}

func validateInGoPath(name string) error {
	gp, err := envy.MustGet("GOPATH")
	if err != nil {
		fmt.Println(noGoPath)
		os.Exit(-1)
	}

	root, err := rootPath(name)
	if err != nil {
		return err
	}

	var gpMultiple []string

	if runtime.GOOS == "windows" {
		gpMultiple = strings.Split(gp, ";") // Windows uses a different separator
	} else {
		gpMultiple = strings.Split(gp, ":")
	}
	gpMultipleLen := len(gpMultiple)
	foundInPath := false

	for i := 0; i < gpMultipleLen; i++ {
		if strings.HasPrefix(root, filepath.Join(gpMultiple[i], "src")) {
			foundInPath = true
			break
		}
	}

	if !foundInPath {
		u, err := user.Current()
		if err != nil {
			return err
		}
		t, err := velvet.Render(notInGoWorkspace, velvet.NewContextWith(map[string]interface{}{
			"name":     name,
			"gopath":   gp,
			"current":  root,
			"username": u.Username,
		}))
		if err != nil {
			return err
		}
		fmt.Println(t)
		os.Exit(-1)
	}
	return nil
}

func goPath(root string) string {
	var gpMultiple []string
	gp := os.Getenv("GOPATH")

	if runtime.GOOS == "windows" {
		gpMultiple = strings.Split(gp, ";") // Windows uses a different separator
	} else {
		gpMultiple = strings.Split(gp, ":")
	}
	gpMultipleLen := len(gpMultiple)
	path := ""

	for i := 0; i < gpMultipleLen; i++ {
		if strings.HasPrefix(root, filepath.Join(gpMultiple[i], "src")) {
			path = gpMultiple[i]
			break
		}
	}
	return path
}

func rootPath(name string) (string, error) {
	pwd, err := os.Getwd()
	if err != nil {
		return "", err
	}
	rootPath := filepath.Join(pwd, name)
	return rootPath, nil
}

func packagePath(rootPath string) string {
	gosrcpath := strings.Replace(filepath.Join(goPath(rootPath), "src"), "\\", "/", -1)
	rootPath = strings.Replace(rootPath, "\\", "/", -1)
	return strings.Replace(rootPath, gosrcpath+"/", "", 2)
}

func genNewFiles(name, rootPath string) error {
	packagePath := packagePath(rootPath)

	data := map[string]interface{}{
		"name":        name,
		"titleName":   inflect.Titleize(name),
		"packagePath": packagePath,
		"actionsPath": packagePath + "/actions",
		"modelsPath":  packagePath + "/models",
		"withPop":     !skipPop,
		"withWebpack": !skipWebpack,
		"dbType":      dbType,
		"version":     Version,
		"ciProvider":  ciProvider,
	}

	g := newAppGenerator(data)
	return g.Run(rootPath, data)
}

func init() {
	RootCmd.AddCommand(newCmd)
	newCmd.Flags().BoolVarP(&force, "force", "f", false, "delete and remake if the app already exists")
	newCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "verbosely print out the go get/install commands")
	newCmd.Flags().BoolVar(&skipPop, "skip-pop", false, "skips adding pop/soda to your app")
	newCmd.Flags().BoolVar(&skipWebpack, "skip-webpack", false, "skips adding Webpack to your app")
	newCmd.Flags().StringVar(&dbType, "db-type", "postgres", "specify the type of database you want to use [postgres, mysql, sqlite3]")
	newCmd.Flags().StringVar(&ciProvider, "ci-provider", "none", "specify the type of ci file you would like buffalo to generate [none, travis]")
}

const notInGoWorkspace = `Oops! It would appear that you are not in your Go Workspace.

Your $GOPATH is set to "{{gopath}}".

You are currently in "{{current}}".

The standard location for putting Go projects is something along the lines of "$GOPATH/src/github.com/{{username}}/{{name}}" (adjust accordingly).

We recommend you go to "$GOPATH/src/github.com/{{username}}/" and try "buffalo new {{name}}" again.`

const noGoPath = `You do not have a $GOPATH set. In order to work with Go, you must set up your $GOPATH and your Go Workspace.

We recommend reading this tutorial on setting everything up: https://www.goinggo.net/2016/05/installing-go-and-your-workspace.html

When you're ready come back and try again. Don't worry, Buffalo will be right here waiting for you. :)`
<|endoftext|>


<commit_before>package timber

import (
	"bufio"
	"io"
	"log"
)

// Use this of you need some buffering, or not
type BufferedWriter struct {
	buf    *bufio.Writer
	writer io.WriteCloser
}

func NewBufferedWriter(writer io.WriteCloser) *BufferedWriter {
	bw := new(BufferedWriter)
	bw.writer = writer
	bw.buf = bufio.NewWriter(writer)
	return bw
}

func (bw *BufferedWriter) LogWrite(msg string) {
	_, err := bw.buf.Write([]byte(msg))
	if err != nil {
		// uh-oh... what do i do if logging fails; punt!
		log.Printf("TIMBER! epic fail: %v", err)
	}
}

// Force flush the buffer
func (bw *BufferedWriter) Flush() {
	bw.buf.Flush()
}

func (bw *BufferedWriter) Close() {
	bw.buf.Flush()
	bw.writer.Close()
}
<commit_msg>make buffered_write safe for gomaxprox > 1<commit_after>package timber

import (
	"bufio"
	"io"
	"log"
)

// Use this of you need some buffering, or not
type BufferedWriter struct {
	buf    *bufio.Writer
	writer io.WriteCloser
	mc chan string
	fc chan int
}

func NewBufferedWriter(writer io.WriteCloser) *BufferedWriter {
	bw := new(BufferedWriter)
	bw.writer = writer
	bw.buf = bufio.NewWriter(writer)
	bw.mc = make(chan string)
	bw.fc = make(chan int)
	go bw.writeLoop()
	return bw
}

func (bw *BufferedWriter) writeLoop() {
	for {
		select {
		case msg, ok := <- bw.mc:
			if !ok {
				bw.buf.Flush()
				bw.writer.Close()
				return
			}
			_, err := bw.buf.Write([]byte(msg))
			if err != nil {
				// uh-oh... what do i do if logging fails; punt!
				log.Printf("TIMBER! epic fail: %v", err)
			}
		case <- bw.fc:
			bw.buf.Flush()
		}
	}
}

func (bw *BufferedWriter) LogWrite(msg string) {
	bw.mc <- msg
}

// Force flush the buffer
func (bw *BufferedWriter) Flush() {
	bw.fc <- 1
}

func (bw *BufferedWriter) Close() {
	close(bw.mc)
}
<|endoftext|>


<commit_before>package main

const controllerTmpl = `
func Get{{ pluralize .Name }}(c *gin.Context) {
	db := db.DBInstance(c)
	fields := c.DefaultQuery("fields", "*")
	var {{ pluralize (tolower .Name) }} []models.{{ .Name }}
	db.Select(fields).Find(&{{ pluralize (tolower .Name) }})
	c.JSON(200, {{ pluralize (tolower .Name) }})
}

func Get{{ .Name }}(c. *gin.Context) {
	db := db.DBInstance(c)
	id := c.Params.ByName("id")
	fields := c.DefaultQuery("fields", "*")
	var {{ tolower .Name }} models.{{ .Name }}
	err := db.Select(fields).First(&{{ tolower .Name }}, id).Error

	if err != nil {
		content := gin.H{"error": "{{ tolower .Name }} with id#" + id + " not found"}
		c.JSON(404, content)
		return
	}

	c.JSON(200, {{ tolower .Name }})
}

func Create{{ .Name }}(c *gin.Context) {
	db := db.DBInstance(c)
	var {{ tolower .Name }} models.{{ .Name }}
	c.Bind(&{{ tolower .Name }})
	if db.Create(&{{ tolower .Name }}).Error != nil {
		content := gin.H{"error": "error occured"}
		c.JSON(500, content)
		return
	}
	c.JSON(201, {{ tolower .Name }})
}

func Update{{ .Name }}(c *gin.Context) {
	db := db.DBInstance(c)
	id := c.Params.ByName("id")
	var {{ tolower .Name }} models.{{ .Name }}
	if db.First(&{{ tolower .Name }}, id).Error != nil {
		content := gin.H{"error": "{{ tolower .Name }} with id#" + id + " not found"}
		c.JSON(404, content)
		return
	}
	c.Bind(&{{ tolower .Name }})
	db.Save(&{{ tolower .Name }})
	c.JSON(200, {{ tolower .Name }})
}

func Delete{{ .Name }}(c *gin.Context) {
	db := db.DBInstance(c)
	id := c.Params.ByName("id")
	var {{ tolower .Name }} models.{{ .Name }}
	if db.First(&{{ tolower .Name }}, id).Error != nil {
		content := gin.H{"error": "{{ tolower .Name }} with id#" + id + " not found"}
		c.JSON(404, content)
		return
	}
	db.Delete(&{{ tolower .Name }})
	c.Writer.WriteHeader(http.StatusNoContent)
}
`

const routerTmpl = `
//{{ .Name }} API
		api.GET("/{{ pluralize (tolower .Name) }}", controllers.Get{{ pluralize .Name }})
		api.GET("/{{ pluralize (tolower .Name) }}/:id", controllers.Get{{ .Name }})
		api.POST("/{{ pluralize (tolower .Name) }}", controllers.Create{{ .Name }})
		api.PUT("/{{ pluralize (tolower .Name) }}/:id", controllers.Update{{ .Name }})
		api.DELETE("/{{ pluralize (tolower .Name) }}/:id", controllers.Delete{{ .Name }})
`
<commit_msg>Modify template<commit_after>package main

const controllerTmpl = `func Get{{ pluralize .Name }}(c *gin.Context) {
	db := db.DBInstance(c)
	fields := c.DefaultQuery("fields", "*")
	var {{ pluralize (tolower .Name) }} []models.{{ .Name }}
	db.Select(fields).Find(&{{ pluralize (tolower .Name) }})
	c.JSON(200, {{ pluralize (tolower .Name) }})
}

func Get{{ .Name }}(c. *gin.Context) {
	db := db.DBInstance(c)
	id := c.Params.ByName("id")
	fields := c.DefaultQuery("fields", "*")
	var {{ tolower .Name }} models.{{ .Name }}
	err := db.Select(fields).First(&{{ tolower .Name }}, id).Error

	if err != nil {
		content := gin.H{"error": "{{ tolower .Name }} with id#" + id + " not found"}
		c.JSON(404, content)
		return
	}

	c.JSON(200, {{ tolower .Name }})
}

func Create{{ .Name }}(c *gin.Context) {
	db := db.DBInstance(c)
	var {{ tolower .Name }} models.{{ .Name }}
	c.Bind(&{{ tolower .Name }})
	if db.Create(&{{ tolower .Name }}).Error != nil {
		content := gin.H{"error": "error occured"}
		c.JSON(500, content)
		return
	}
	c.JSON(201, {{ tolower .Name }})
}

func Update{{ .Name }}(c *gin.Context) {
	db := db.DBInstance(c)
	id := c.Params.ByName("id")
	var {{ tolower .Name }} models.{{ .Name }}
	if db.First(&{{ tolower .Name }}, id).Error != nil {
		content := gin.H{"error": "{{ tolower .Name }} with id#" + id + " not found"}
		c.JSON(404, content)
		return
	}
	c.Bind(&{{ tolower .Name }})
	db.Save(&{{ tolower .Name }})
	c.JSON(200, {{ tolower .Name }})
}

func Delete{{ .Name }}(c *gin.Context) {
	db := db.DBInstance(c)
	id := c.Params.ByName("id")
	var {{ tolower .Name }} models.{{ .Name }}
	if db.First(&{{ tolower .Name }}, id).Error != nil {
		content := gin.H{"error": "{{ tolower .Name }} with id#" + id + " not found"}
		c.JSON(404, content)
		return
	}
	db.Delete(&{{ tolower .Name }})
	c.Writer.WriteHeader(http.StatusNoContent)
}
`

const routerTmpl = `//{{ .Name }} API
		api.GET("/{{ pluralize (tolower .Name) }}", controllers.Get{{ pluralize .Name }})
		api.GET("/{{ pluralize (tolower .Name) }}/:id", controllers.Get{{ .Name }})
		api.POST("/{{ pluralize (tolower .Name) }}", controllers.Create{{ .Name }})
		api.PUT("/{{ pluralize (tolower .Name) }}/:id", controllers.Update{{ .Name }})
		api.DELETE("/{{ pluralize (tolower .Name) }}/:id", controllers.Delete{{ .Name }})
`
<|endoftext|>