<commit_before>package libkbfs

import (
	"crypto/tls"
	"crypto/x509"
	"errors"
	"net"
	"sync"
	"time"

	"github.com/cenkalti/backoff"
	"github.com/keybase/client/go/libkb"
	keybase1 "github.com/keybase/client/protocol/go"
	"github.com/maxtaco/go-framed-msgpack-rpc/rpc2"
	"golang.org/x/net/context"
)

// ConnectionHandler is the callback interface for interacting with the connection.
type ConnectionHandler interface {
	// OnConnect is called immediately after a connection has been established.
	// An implementation would likely log something and/or perform authentication.
	OnConnect(context.Context, *Connection, keybase1.GenericClient) error

	// OnConnectError is called whenever there is an error during connection.
	OnConnectError(err error, reconnectThrottleDuration time.Duration)
}

// ConnectionTransportTLS is a ConnectionTransport implementation that uses TLS+rpc2.
type ConnectionTransportTLS struct {
	config          Config
	unwrapErrFunc   rpc2.UnwrapErrorFunc
	transport       *rpc2.Transport
	stagedTransport *rpc2.Transport
	mutex           sync.Mutex // protects transport
}

// Test that ConnectionTransportTLS fully implements the ConnectionTransport interface.
var _ ConnectionTransport = (*ConnectionTransportTLS)(nil)

// Dial is an implementation of the ConnectionTransport interface.
func (ct *ConnectionTransportTLS) Dial(ctx context.Context, srvAddr string) (
	keybase1.GenericClient, error) {
	var conn net.Conn
	var err error
	err = runUnlessCanceled(ctx, func() error {
		// load CA certificate
		certs := x509.NewCertPool()
		if !certs.AppendCertsFromPEM(ct.config.CACert()) {
			return errors.New("Unable to load CA certificate")
		}
		// connect
		config := tls.Config{RootCAs: certs}
		conn, err = tls.Dial("tcp", srvAddr, &config)
		return err
	})
	if err != nil {
		return nil, err
	}

	ct.stagedTransport = rpc2.NewTransport(conn, libkb.NewRPCLogFactory(), libkb.WrapError)
	client := rpc2.NewClient(ct.stagedTransport, ct.unwrapErrFunc)
	return client, nil
}

// IsConnected is an implementation of the ConnectionTransport interface.
func (ct *ConnectionTransportTLS) IsConnected() bool {
	ct.mutex.Lock()
	defer ct.mutex.Unlock()
	return ct.transport != nil && ct.transport.IsConnected()
}

// Finalize is an implementation of the ConnectionTransport interface.
func (ct *ConnectionTransportTLS) Finalize() {
	ct.mutex.Lock()
	defer ct.mutex.Unlock()
	ct.transport = ct.stagedTransport
	ct.stagedTransport = nil
}

// Connection encapsulates all client connection handling.
type Connection struct {
	config     Config
	srvAddr    string
	handler    ConnectionHandler
	transport  ConnectionTransport
	ctx        context.Context    // used for canceling reconnect loops
	cancelFunc context.CancelFunc // ^same

	mutex         sync.Mutex // protects: client and reconnectChan
	client        keybase1.GenericClient
	reconnectChan chan struct{}
}

// NewConnection returns a newly connected connection.
func NewConnection(ctx context.Context, config Config, srvAddr string,
	handler ConnectionHandler, errFunc rpc2.UnwrapErrorFunc) *Connection {
	transport := &ConnectionTransportTLS{config: config, unwrapErrFunc: errFunc}
	return newConnectionWithTransport(ctx, config, srvAddr, handler, transport)
}

// Separate from NewConnection to allow for unit testing.
func newConnectionWithTransport(ctx context.Context, config Config, srvAddr string,
	handler ConnectionHandler, transport ConnectionTransport) *Connection {
	connCtx, cancelFunc := context.WithCancel(context.Background())
	connection := &Connection{
		srvAddr:    srvAddr,
		config:     config,
		handler:    handler,
		ctx:        connCtx,
		cancelFunc: cancelFunc,
		transport:  transport,
	}
	connection.getReconnectChan() // start connecting
	return connection
}

// connect performs the actual connect() and rpc setup.
func (c *Connection) connect(ctx context.Context) error {
	// connect
	client, err := c.transport.Dial(ctx, c.srvAddr)
	if err != nil {
		return err
	}

	// call the connect handler
	err = c.handler.OnConnect(ctx, c, client)
	if err != nil {
		return err
	}

	// set the client for other callers.
	// we wait to do this so the handler has time to do
	// any setup required, e.g. authenticate.
	c.mutex.Lock()
	defer c.mutex.Unlock()
	c.client = client
	c.transport.Finalize()

	return nil
}

// DoCommand executes the specific rpc command wrapped in rpcFunc.
func (c *Connection) DoCommand(ctx context.Context, rpcFunc func() error) error {
	for {
		// we may or may not be in the process of reconnecting.
		// if so we'll block here unless canceled by the caller.
		err := c.waitForConnection(ctx)
		if err != nil {
			return err
		}
		// try the rpc call. this can also be canceled by the caller.
		err = runUnlessCanceled(ctx, rpcFunc)
		// check to see if we need to retry it.
		if !c.checkForRetry(err) {
			return err
		}
	}
}

// Blocks until a connnection is ready for use or the context is canceled.
func (c *Connection) waitForConnection(ctx context.Context) error {
	if c.IsConnected() {
		// already connected
		return nil
	}
	// kick-off a connection and wait for it to complete
	// or for the caller to cancel.
	reconnectChan := c.getReconnectChan()
	select {
	case <-ctx.Done():
		// caller canceled
		return ctx.Err()
	case <-reconnectChan:
		// reconnect complete
		return nil
	}
}

// Returns true if the error indicates we should retry the command.
func (c *Connection) checkForRetry(err error) bool {
	if err == nil {
		return false
	}
	_, disconnected := err.(rpc2.DisconnectedError)
	_, eof := err.(rpc2.EofError)
	return disconnected || eof
}

// IsConnected returns true if the connection is connected.
func (c *Connection) IsConnected() bool {
	return c.transport.IsConnected()
}

// This will either kick-off a new reconnection attempt or wait for an
// existing attempt. Returns the channel associated with an attempt.
func (c *Connection) getReconnectChan() chan struct{} {
	c.mutex.Lock()
	defer c.mutex.Unlock()
	if c.reconnectChan == nil {
		c.reconnectChan = make(chan struct{}, 20)
		go c.doReconnect(c.reconnectChan)
	}
	return c.reconnectChan
}

// doReconnect attempts a reconnection.
func (c *Connection) doReconnect(reconnectChan chan struct{}) {
	// retry w/exponential backoff
	backoff.RetryNotify(func() error {
		// try to connect
		err := c.connect(c.ctx)
		// context was canceled by Shutdown()
		if c.ctx.Err() != nil {
			c.handler = nil // drop the circular reference
			// short-circuit Retry
			return nil
		}
		return err
	}, backoff.NewExponentialBackOff(),
		// give the caller a chance to log any other error or adjust state
		c.handler.OnConnectError)

	// close the reconnect channel to signal we're connected.
	c.mutex.Lock()
	defer c.mutex.Unlock()
	close(reconnectChan)
	c.reconnectChan = nil
}

// GetClient is called to retrieve an rpc client suitable for use by the caller.
func (c *Connection) GetClient() keybase1.GenericClient {
	c.mutex.Lock()
	defer c.mutex.Unlock()
	return c.client
}

// Shutdown cancels any reconnect loop in progress.
func (c *Connection) Shutdown() {
	c.cancelFunc() // cancel any reconnect loop
	// TODO: I think it would be ideal for this to also close
	// the transport but the rpc2 package doesn't appear to
	// expose that ability at the moment.
}
<commit_msg>connection: don't store context in the struct<commit_after>package libkbfs

import (
	"crypto/tls"
	"crypto/x509"
	"errors"
	"net"
	"sync"
	"time"

	"github.com/cenkalti/backoff"
	"github.com/keybase/client/go/libkb"
	keybase1 "github.com/keybase/client/protocol/go"
	"github.com/maxtaco/go-framed-msgpack-rpc/rpc2"
	"golang.org/x/net/context"
)

// ConnectionHandler is the callback interface for interacting with the connection.
type ConnectionHandler interface {
	// OnConnect is called immediately after a connection has been established.
	// An implementation would likely log something and/or perform authentication.
	OnConnect(context.Context, *Connection, keybase1.GenericClient) error

	// OnConnectError is called whenever there is an error during connection.
	OnConnectError(err error, reconnectThrottleDuration time.Duration)
}

// ConnectionTransportTLS is a ConnectionTransport implementation that uses TLS+rpc2.
type ConnectionTransportTLS struct {
	config          Config
	unwrapErrFunc   rpc2.UnwrapErrorFunc
	transport       *rpc2.Transport
	stagedTransport *rpc2.Transport
	mutex           sync.Mutex // protects transport
}

// Test that ConnectionTransportTLS fully implements the ConnectionTransport interface.
var _ ConnectionTransport = (*ConnectionTransportTLS)(nil)

// Dial is an implementation of the ConnectionTransport interface.
func (ct *ConnectionTransportTLS) Dial(ctx context.Context, srvAddr string) (
	keybase1.GenericClient, error) {
	var conn net.Conn
	var err error
	err = runUnlessCanceled(ctx, func() error {
		// load CA certificate
		certs := x509.NewCertPool()
		if !certs.AppendCertsFromPEM(ct.config.CACert()) {
			return errors.New("Unable to load CA certificate")
		}
		// connect
		config := tls.Config{RootCAs: certs}
		conn, err = tls.Dial("tcp", srvAddr, &config)
		return err
	})
	if err != nil {
		return nil, err
	}

	ct.stagedTransport = rpc2.NewTransport(conn, libkb.NewRPCLogFactory(), libkb.WrapError)
	client := rpc2.NewClient(ct.stagedTransport, ct.unwrapErrFunc)
	return client, nil
}

// IsConnected is an implementation of the ConnectionTransport interface.
func (ct *ConnectionTransportTLS) IsConnected() bool {
	ct.mutex.Lock()
	defer ct.mutex.Unlock()
	return ct.transport != nil && ct.transport.IsConnected()
}

// Finalize is an implementation of the ConnectionTransport interface.
func (ct *ConnectionTransportTLS) Finalize() {
	ct.mutex.Lock()
	defer ct.mutex.Unlock()
	ct.transport = ct.stagedTransport
	ct.stagedTransport = nil
}

// Connection encapsulates all client connection handling.
type Connection struct {
	config    Config
	srvAddr   string
	handler   ConnectionHandler
	transport ConnectionTransport

	mutex         sync.Mutex // protects: client, reconnectChan, and cancelFunc
	client        keybase1.GenericClient
	reconnectChan chan struct{}
	cancelFunc    context.CancelFunc // used to cancel the reconnect loop
}

// NewConnection returns a newly connected connection.
func NewConnection(ctx context.Context, config Config, srvAddr string,
	handler ConnectionHandler, errFunc rpc2.UnwrapErrorFunc) *Connection {
	transport := &ConnectionTransportTLS{config: config, unwrapErrFunc: errFunc}
	return newConnectionWithTransport(ctx, config, srvAddr, handler, transport)
}

// Separate from NewConnection to allow for unit testing.
func newConnectionWithTransport(ctx context.Context, config Config, srvAddr string,
	handler ConnectionHandler, transport ConnectionTransport) *Connection {
	connection := &Connection{
		srvAddr:   srvAddr,
		config:    config,
		handler:   handler,
		transport: transport,
	}
	connection.getReconnectChan() // start connecting
	return connection
}

// connect performs the actual connect() and rpc setup.
func (c *Connection) connect(ctx context.Context) error {
	// connect
	client, err := c.transport.Dial(ctx, c.srvAddr)
	if err != nil {
		return err
	}

	// call the connect handler
	err = c.handler.OnConnect(ctx, c, client)
	if err != nil {
		return err
	}

	// set the client for other callers.
	// we wait to do this so the handler has time to do
	// any setup required, e.g. authenticate.
	c.mutex.Lock()
	defer c.mutex.Unlock()
	c.client = client
	c.transport.Finalize()

	return nil
}

// DoCommand executes the specific rpc command wrapped in rpcFunc.
func (c *Connection) DoCommand(ctx context.Context, rpcFunc func() error) error {
	for {
		// we may or may not be in the process of reconnecting.
		// if so we'll block here unless canceled by the caller.
		err := c.waitForConnection(ctx)
		if err != nil {
			return err
		}
		// try the rpc call. this can also be canceled by the caller.
		err = runUnlessCanceled(ctx, rpcFunc)
		// check to see if we need to retry it.
		if !c.checkForRetry(err) {
			return err
		}
	}
}

// Blocks until a connnection is ready for use or the context is canceled.
func (c *Connection) waitForConnection(ctx context.Context) error {
	if c.IsConnected() {
		// already connected
		return nil
	}
	// kick-off a connection and wait for it to complete
	// or for the caller to cancel.
	reconnectChan := c.getReconnectChan()
	select {
	case <-ctx.Done():
		// caller canceled
		return ctx.Err()
	case <-reconnectChan:
		// reconnect complete
		return nil
	}
}

// Returns true if the error indicates we should retry the command.
func (c *Connection) checkForRetry(err error) bool {
	if err == nil {
		return false
	}
	_, disconnected := err.(rpc2.DisconnectedError)
	_, eof := err.(rpc2.EofError)
	return disconnected || eof
}

// IsConnected returns true if the connection is connected.
func (c *Connection) IsConnected() bool {
	return c.transport.IsConnected()
}

// This will either kick-off a new reconnection attempt or wait for an
// existing attempt. Returns the channel associated with an attempt.
func (c *Connection) getReconnectChan() chan struct{} {
	c.mutex.Lock()
	defer c.mutex.Unlock()
	if c.reconnectChan == nil {
		var ctx context.Context
		// for canceleing the reconnect loop via Shutdown()
		ctx, c.cancelFunc = context.WithCancel(context.Background())
		c.reconnectChan = make(chan struct{}, 20)
		go c.doReconnect(ctx, c.reconnectChan)
	}
	return c.reconnectChan
}

// doReconnect attempts a reconnection.
func (c *Connection) doReconnect(ctx context.Context, reconnectChan chan struct{}) {
	// retry w/exponential backoff
	backoff.RetryNotify(func() error {
		// try to connect
		err := c.connect(ctx)
		// context was canceled by Shutdown()
		if ctx.Err() != nil {
			c.handler = nil // drop the circular reference
			// short-circuit Retry
			return nil
		}
		return err
	}, backoff.NewExponentialBackOff(),
		// give the caller a chance to log any other error or adjust state
		c.handler.OnConnectError)

	// close the reconnect channel to signal we're connected.
	c.mutex.Lock()
	defer c.mutex.Unlock()
	close(reconnectChan)
	c.reconnectChan = nil
	c.cancelFunc = nil
}

// GetClient is called to retrieve an rpc client suitable for use by the caller.
func (c *Connection) GetClient() keybase1.GenericClient {
	c.mutex.Lock()
	defer c.mutex.Unlock()
	return c.client
}

// Shutdown cancels any reconnect loop in progress.
// Calling this invalidates the connection object.
func (c *Connection) Shutdown() {
	c.mutex.Lock()
	defer c.mutex.Unlock()
	// cancel any reconnect loop
	if c.cancelFunc != nil {
		c.cancelFunc()
	}
	// TODO: I think it would be ideal for this to also close
	// the transport but the rpc2 package doesn't appear to
	// expose that ability at the moment.
}
<|endoftext|>


<commit_before>package cion

import (
	"code.google.com/p/snappy-go/snappy"
	"encoding/binary"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/boltdb/bolt"
	"io"
)

var (
	JobsBucket    = []byte("jobs")
	JobRefsBucket = []byte("jobrefs")
)

type BoltJobStore struct {
	db *bolt.DB
}

type BoltJobLogger struct {
	db  *bolt.DB
	ref boltJobRef
}

// buckets provides references to the various Bolt buckets where job data is stored.
type buckets struct {
	Jobs  *bolt.Bucket
	Owner *bolt.Bucket
	Repo  *bolt.Bucket
	Logs  *bolt.Bucket
}

// boltJobRef is a reference to a job in a specific owner/repo/branch bucket.
type boltJobRef struct {
	Owner  string
	Repo   string
	Number uint64
}

func NewBoltJobStore(path string) (*BoltJobStore, error) {
	db, err := bolt.Open(path, 0600, nil)
	if err != nil {
		return nil, err
	}

	return &BoltJobStore{db: db}, nil
}

func (s *BoltJobStore) GetByNumber(owner, repo string, number uint64) (*Job, error) {
	j := &Job{}

	ref := boltJobRef{
		Owner:  owner,
		Repo:   repo,
		Number: number,
	}

	err := s.db.View(func(tx *bolt.Tx) error {
		b, err := getBuckets(ref, tx)
		if err != nil {
			return err
		}

		key := Uint64ToBytes(number)
		return json.Unmarshal(b.Repo.Get(key), j)
	})
	if err != nil {
		return nil, err
	}

	return j, nil
}

func (s *BoltJobStore) ListOwners() ([]string, error) {
	var l []string

	if err := s.db.View(func(tx *bolt.Tx) error {
		jb := tx.Bucket(JobsBucket)
		if jb == nil {
			return errors.New("jobs bucket doesn't exist")
		}

		return jb.ForEach(func(key, val []byte) error {
			l = append(l, string(key))
			return nil
		})
	}); err != nil {
		return nil, err
	}

	return l, nil
}

func (s *BoltJobStore) ListRepos(owner string) ([]string, error) {
	var l []string

	if err := s.db.View(func(tx *bolt.Tx) error {
		jb := tx.Bucket(JobsBucket)
		if jb == nil {
			return errors.New("jobs bucket doesn't exist")
		}

		ob := jb.Bucket([]byte(owner))
		if ob == nil {
			return errors.New("owner bucket doesn't exist")
		}

		return ob.ForEach(func(key, val []byte) error {
			l = append(l, string(key))
			return nil
		})
	}); err != nil {
		return nil, err
	}

	return l, nil
}

func (s *BoltJobStore) List(owner, repo string) ([]*Job, error) {
	var l []*Job

	ref := boltJobRef{
		Owner: owner,
		Repo:  repo,
	}

	if err := s.db.View(func(tx *bolt.Tx) error {
		b, err := getBuckets(ref, tx)
		if err != nil {
			return err
		}

		return b.Repo.ForEach(func(key, val []byte) error {
			j := &Job{}

			if len(val) == 0 {
				return nil
			}

			if err := json.Unmarshal(val, j); err != nil {
				return err
			}

			l = append(l, j)
			return nil
		})
	}); err != nil {
		return nil, err
	}

	return l, nil
}

func (s *BoltJobStore) GetLogger(j *Job) JobLogger {
	return BoltJobLogger{
		db: s.db,
		ref: boltJobRef{
			Owner:  j.Owner,
			Repo:   j.Repo,
			Number: j.Number,
		},
	}
}

func (l BoltJobLogger) Write(p []byte) (int, error) {
	if err := l.db.Update(func(tx *bolt.Tx) error {
		b, err := getBuckets(l.ref, tx)
		if err != nil {
			return err
		}

		i, err := b.Logs.NextSequence()
		if err != nil {
			return err
		}

		key := Uint64ToBytes(i)
		val, err := snappy.Encode(nil, p)
		if err != nil {
			return err
		}

		return b.Logs.Put(key, val)
	}); err != nil {
		return 0, err
	}

	return len(p), nil
}

func (l BoltJobLogger) WriteTo(w io.Writer) (int64, error) {
	var n int64

	return n, l.db.View(func(tx *bolt.Tx) error {
		b, err := getBuckets(l.ref, tx)
		if err != nil {
			return err
		}

		return b.Logs.ForEach(func(key, val []byte) error {
			s, err := snappy.Decode(nil, val)
			if err != nil {
				return err
			}

			c, err := w.Write(s)
			n = n + int64(c)

			return err
		})
	})
}

func (l BoltJobLogger) WriteStep(name string) error {
	s := fmt.Sprintln("---", name, "---")
	_, err := l.Write([]byte(s))
	return err
}

// Save writes job data to various buckets in the Bolt database. We use this nesting pattern
// for buckets:
//
//    jobs -> (owner) -> (repo) -> logs_(job number)
//
// The actual data for a job is saved to the bucket for its repo, and its logs are stored in
// the logs_<number> sub-bucket.
func (s *BoltJobStore) Save(j *Job) error {
	ref := boltJobRef{
		Owner:  j.Owner,
		Repo:   j.Repo,
		Number: j.Number,
	}

	return s.db.Update(func(tx *bolt.Tx) error {
		b, err := getBuckets(ref, tx)
		if err != nil {
			return err
		}

		if j.Number == 0 {
			// assign the job an incrementing build number
			n, err := b.Repo.NextSequence()
			if err != nil {
				return err
			}

			j.Number = n
		}

		key := Uint64ToBytes(j.Number)
		val, err := json.Marshal(j)
		if err != nil {
			return err
		}

		return b.Repo.Put(key, val)
	})
}

func getBuckets(ref boltJobRef, tx *bolt.Tx) (*buckets, error) {
	var jb, ob, rb, lb *bolt.Bucket
	var err error

	if tx.Writable() {
		jb, err = tx.CreateBucketIfNotExists(JobsBucket)
	} else {
		jb = tx.Bucket(JobsBucket)
	}
	if err != nil {
		return nil, err
	} else if jb == nil {
		return nil, errors.New("jobs bucket doesn't exist")
	}

	if tx.Writable() {
		ob, err = jb.CreateBucketIfNotExists([]byte(ref.Owner))
	} else {
		ob = jb.Bucket([]byte(ref.Owner))
	}
	if err != nil {
		return nil, err
	} else if ob == nil {
		return nil, errors.New("owner bucket doesn't exist for ref: " + fmt.Sprint(ref))
	}

	if tx.Writable() {
		rb, err = ob.CreateBucketIfNotExists([]byte(ref.Repo))
	} else {
		rb = ob.Bucket([]byte(ref.Repo))
	}
	if err != nil {
		return nil, err
	} else if rb == nil {
		return nil, errors.New("repo bucket doesn't exist for ref: " + fmt.Sprint(ref))
	}

	if ref.Number != 0 {
		lbn := "logs_" + string(ref.Number)

		if tx.Writable() {
			lb, err = rb.CreateBucketIfNotExists([]byte(lbn))
		} else {
			lb = rb.Bucket([]byte(lbn))
		}
		if err != nil {
			return nil, err
		} else if lb == nil {
			return nil, errors.New("logs bucket doesn't exist for ref: " + fmt.Sprint(ref))
		}
	}

	return &buckets{
		Jobs:  jb,
		Owner: ob,
		Repo:  rb,
		Logs:  lb,
	}, nil
}

func Uint64ToBytes(x uint64) []byte {
	b := make([]byte, 8)
	binary.BigEndian.PutUint64(b, x)

	return b
}
<commit_msg>bolt: Return job list in reverse order<commit_after>package cion

import (
	"code.google.com/p/snappy-go/snappy"
	"encoding/binary"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/boltdb/bolt"
	"io"
)

var (
	JobsBucket    = []byte("jobs")
	JobRefsBucket = []byte("jobrefs")
)

type BoltJobStore struct {
	db *bolt.DB
}

type BoltJobLogger struct {
	db  *bolt.DB
	ref boltJobRef
}

// buckets provides references to the various Bolt buckets where job data is stored.
type buckets struct {
	Jobs  *bolt.Bucket
	Owner *bolt.Bucket
	Repo  *bolt.Bucket
	Logs  *bolt.Bucket
}

// boltJobRef is a reference to a job in a specific owner/repo/branch bucket.
type boltJobRef struct {
	Owner  string
	Repo   string
	Number uint64
}

func NewBoltJobStore(path string) (*BoltJobStore, error) {
	db, err := bolt.Open(path, 0600, nil)
	if err != nil {
		return nil, err
	}

	return &BoltJobStore{db: db}, nil
}

func (s *BoltJobStore) GetByNumber(owner, repo string, number uint64) (*Job, error) {
	j := &Job{}

	ref := boltJobRef{
		Owner:  owner,
		Repo:   repo,
		Number: number,
	}

	err := s.db.View(func(tx *bolt.Tx) error {
		b, err := getBuckets(ref, tx)
		if err != nil {
			return err
		}

		key := Uint64ToBytes(number)
		return json.Unmarshal(b.Repo.Get(key), j)
	})
	if err != nil {
		return nil, err
	}

	return j, nil
}

func (s *BoltJobStore) ListOwners() ([]string, error) {
	var l []string

	if err := s.db.View(func(tx *bolt.Tx) error {
		jb := tx.Bucket(JobsBucket)
		if jb == nil {
			return errors.New("jobs bucket doesn't exist")
		}

		return jb.ForEach(func(key, val []byte) error {
			l = append(l, string(key))
			return nil
		})
	}); err != nil {
		return nil, err
	}

	return l, nil
}

func (s *BoltJobStore) ListRepos(owner string) ([]string, error) {
	var l []string

	if err := s.db.View(func(tx *bolt.Tx) error {
		jb := tx.Bucket(JobsBucket)
		if jb == nil {
			return errors.New("jobs bucket doesn't exist")
		}

		ob := jb.Bucket([]byte(owner))
		if ob == nil {
			return errors.New("owner bucket doesn't exist")
		}

		return ob.ForEach(func(key, val []byte) error {
			l = append(l, string(key))
			return nil
		})
	}); err != nil {
		return nil, err
	}

	return l, nil
}

func (s *BoltJobStore) List(owner, repo string) ([]*Job, error) {
	var l []*Job

	ref := boltJobRef{
		Owner: owner,
		Repo:  repo,
	}

	if err := s.db.View(func(tx *bolt.Tx) error {
		b, err := getBuckets(ref, tx)
		if err != nil {
			return err
		}

		c := b.Repo.Cursor()
		for key, val := c.Last(); key != nil; key, val = c.Prev() {
			j := &Job{}

			if len(val) > 0 {
				if err := json.Unmarshal(val, j); err != nil {
					return err
				}

				l = append(l, j)
			}
		}

		return nil
	}); err != nil {
		return nil, err
	}

	return l, nil
}

func (s *BoltJobStore) GetLogger(j *Job) JobLogger {
	return BoltJobLogger{
		db: s.db,
		ref: boltJobRef{
			Owner:  j.Owner,
			Repo:   j.Repo,
			Number: j.Number,
		},
	}
}

func (l BoltJobLogger) Write(p []byte) (int, error) {
	if err := l.db.Update(func(tx *bolt.Tx) error {
		b, err := getBuckets(l.ref, tx)
		if err != nil {
			return err
		}

		i, err := b.Logs.NextSequence()
		if err != nil {
			return err
		}

		key := Uint64ToBytes(i)
		val, err := snappy.Encode(nil, p)
		if err != nil {
			return err
		}

		return b.Logs.Put(key, val)
	}); err != nil {
		return 0, err
	}

	return len(p), nil
}

func (l BoltJobLogger) WriteTo(w io.Writer) (int64, error) {
	var n int64

	return n, l.db.View(func(tx *bolt.Tx) error {
		b, err := getBuckets(l.ref, tx)
		if err != nil {
			return err
		}

		return b.Logs.ForEach(func(key, val []byte) error {
			s, err := snappy.Decode(nil, val)
			if err != nil {
				return err
			}

			c, err := w.Write(s)
			n = n + int64(c)

			return err
		})
	})
}

func (l BoltJobLogger) WriteStep(name string) error {
	s := fmt.Sprintln("---", name, "---")
	_, err := l.Write([]byte(s))
	return err
}

// Save writes job data to various buckets in the Bolt database. We use this nesting pattern
// for buckets:
//
//    jobs -> (owner) -> (repo) -> logs_(job number)
//
// The actual data for a job is saved to the bucket for its repo, and its logs are stored in
// the logs_<number> sub-bucket.
func (s *BoltJobStore) Save(j *Job) error {
	ref := boltJobRef{
		Owner:  j.Owner,
		Repo:   j.Repo,
		Number: j.Number,
	}

	return s.db.Update(func(tx *bolt.Tx) error {
		b, err := getBuckets(ref, tx)
		if err != nil {
			return err
		}

		if j.Number == 0 {
			// assign the job an incrementing build number
			n, err := b.Repo.NextSequence()
			if err != nil {
				return err
			}

			j.Number = n
		}

		key := Uint64ToBytes(j.Number)
		val, err := json.Marshal(j)
		if err != nil {
			return err
		}

		return b.Repo.Put(key, val)
	})
}

func getBuckets(ref boltJobRef, tx *bolt.Tx) (*buckets, error) {
	var jb, ob, rb, lb *bolt.Bucket
	var err error

	if tx.Writable() {
		jb, err = tx.CreateBucketIfNotExists(JobsBucket)
	} else {
		jb = tx.Bucket(JobsBucket)
	}
	if err != nil {
		return nil, err
	} else if jb == nil {
		return nil, errors.New("jobs bucket doesn't exist")
	}

	if tx.Writable() {
		ob, err = jb.CreateBucketIfNotExists([]byte(ref.Owner))
	} else {
		ob = jb.Bucket([]byte(ref.Owner))
	}
	if err != nil {
		return nil, err
	} else if ob == nil {
		return nil, errors.New("owner bucket doesn't exist for ref: " + fmt.Sprint(ref))
	}

	if tx.Writable() {
		rb, err = ob.CreateBucketIfNotExists([]byte(ref.Repo))
	} else {
		rb = ob.Bucket([]byte(ref.Repo))
	}
	if err != nil {
		return nil, err
	} else if rb == nil {
		return nil, errors.New("repo bucket doesn't exist for ref: " + fmt.Sprint(ref))
	}

	if ref.Number != 0 {
		lbn := "logs_" + string(ref.Number)

		if tx.Writable() {
			lb, err = rb.CreateBucketIfNotExists([]byte(lbn))
		} else {
			lb = rb.Bucket([]byte(lbn))
		}
		if err != nil {
			return nil, err
		} else if lb == nil {
			return nil, errors.New("logs bucket doesn't exist for ref: " + fmt.Sprint(ref))
		}
	}

	return &buckets{
		Jobs:  jb,
		Owner: ob,
		Repo:  rb,
		Logs:  lb,
	}, nil
}

func Uint64ToBytes(x uint64) []byte {
	b := make([]byte, 8)
	binary.BigEndian.PutUint64(b, x)

	return b
}
<|endoftext|>


<commit_before>package mod

import (
	"bytes"
	"errors"
	"io"
	"os"

	proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
	mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"

	chunk "github.com/jbenet/go-ipfs/importer/chunk"
	help "github.com/jbenet/go-ipfs/importer/helpers"
	trickle "github.com/jbenet/go-ipfs/importer/trickle"
	mdag "github.com/jbenet/go-ipfs/merkledag"
	pin "github.com/jbenet/go-ipfs/pin"
	ft "github.com/jbenet/go-ipfs/unixfs"
	uio "github.com/jbenet/go-ipfs/unixfs/io"
	u "github.com/jbenet/go-ipfs/util"
)

var ErrSeekFail = errors.New("failed to seek properly")
var ErrSeekEndNotImpl = errors.New("SEEK_END currently not implemented")
var ErrUnrecognizedWhence = errors.New("unrecognized whence")

// 2MB
var writebufferSize = 1 << 21

var log = u.Logger("dagio")

// DagModifier is the only struct licensed and able to correctly
// perform surgery on a DAG 'file'
// Dear god, please rename this to something more pleasant
type DagModifier struct {
	dagserv mdag.DAGService
	curNode *mdag.Node
	mp      pin.ManualPinner

	splitter   chunk.BlockSplitter
	ctx        context.Context
	readCancel func()

	writeStart uint64
	curWrOff   uint64
	wrBuf      *bytes.Buffer

	read *uio.DagReader
}

func NewDagModifier(ctx context.Context, from *mdag.Node, serv mdag.DAGService, mp pin.ManualPinner, spl chunk.BlockSplitter) (*DagModifier, error) {
	return &DagModifier{
		curNode:  from.Copy(),
		dagserv:  serv,
		splitter: spl,
		ctx:      ctx,
		mp:       mp,
	}, nil
}

// WriteAt will modify a dag file in place
func (dm *DagModifier) WriteAt(b []byte, offset int64) (int, error) {
	// TODO: this is currently VERY inneficient
	// each write that happens at an offset other than the current one causes a
	// flush to disk, and dag rewrite
	if uint64(offset) != dm.curWrOff {
		size, err := dm.Size()
		if err != nil {
			return 0, err
		}
		if offset > size {
			err := dm.expandSparse(offset - size)
			if err != nil {
				return 0, err
			}
		}

		err = dm.Flush()
		if err != nil {
			return 0, err
		}
		dm.writeStart = uint64(offset)
	}

	return dm.Write(b)
}

// A reader that just returns zeros
type zeroReader struct{}

func (zr zeroReader) Read(b []byte) (int, error) {
	for i, _ := range b {
		b[i] = 0
	}
	return len(b), nil
}

// expandSparse grows the file with zero blocks of 4096
// A small blocksize is chosen to aid in deduplication
func (dm *DagModifier) expandSparse(size int64) error {
	spl := chunk.SizeSplitter{4096}
	r := io.LimitReader(zeroReader{}, size)
	blks := spl.Split(r)
	nnode, err := dm.appendData(dm.curNode, blks)
	if err != nil {
		return err
	}
	_, err = dm.dagserv.Add(nnode)
	if err != nil {
		return err
	}
	dm.curNode = nnode
	return nil
}

// Write continues writing to the dag at the current offset
func (dm *DagModifier) Write(b []byte) (int, error) {
	if dm.read != nil {
		dm.read = nil
	}
	if dm.wrBuf == nil {
		dm.wrBuf = new(bytes.Buffer)
	}

	n, err := dm.wrBuf.Write(b)
	if err != nil {
		return n, err
	}
	dm.curWrOff += uint64(n)
	if dm.wrBuf.Len() > writebufferSize {
		err := dm.Flush()
		if err != nil {
			return n, err
		}
	}
	return n, nil
}

func (dm *DagModifier) Size() (int64, error) {
	// TODO: compute size without flushing, should be easy
	err := dm.Flush()
	if err != nil {
		return 0, err
	}

	pbn, err := ft.FromBytes(dm.curNode.Data)
	if err != nil {
		return 0, err
	}

	return int64(pbn.GetFilesize()), nil
}

// Flush writes changes to this dag to disk
func (dm *DagModifier) Flush() error {
	// No buffer? Nothing to do
	if dm.wrBuf == nil {
		return nil
	}

	// If we have an active reader, kill it
	if dm.read != nil {
		dm.read = nil
		dm.readCancel()
	}

	// Number of bytes we're going to write
	buflen := dm.wrBuf.Len()

	// overwrite existing dag nodes
	k, done, err := dm.modifyDag(dm.curNode, dm.writeStart, dm.wrBuf)
	if err != nil {
		return err
	}

	nd, err := dm.dagserv.Get(k)
	if err != nil {
		return err
	}

	dm.curNode = nd

	// need to write past end of current dag
	if !done {
		blks := dm.splitter.Split(dm.wrBuf)
		nd, err = dm.appendData(dm.curNode, blks)
		if err != nil {
			return err
		}

		_, err := dm.dagserv.Add(nd)
		if err != nil {
			return err
		}

		dm.curNode = nd
	}

	dm.writeStart += uint64(buflen)

	dm.wrBuf = nil
	return nil
}

// modifyDag writes the data in 'data' over the data in 'node' starting at 'offset'
// returns the new key of the passed in node and whether or not all the data in the reader
// has been consumed.
func (dm *DagModifier) modifyDag(node *mdag.Node, offset uint64, data io.Reader) (u.Key, bool, error) {
	f, err := ft.FromBytes(node.Data)
	if err != nil {
		return "", false, err
	}

	// If we've reached a leaf node.
	if len(node.Links) == 0 {
		n, err := data.Read(f.Data[offset:])
		if err != nil && err != io.EOF {
			return "", false, err
		}

		// Update newly written node..
		b, err := proto.Marshal(f)
		if err != nil {
			return "", false, err
		}

		nd := &mdag.Node{Data: b}
		k, err := dm.dagserv.Add(nd)
		if err != nil {
			return "", false, err
		}

		// Hey look! we're done!
		var done bool
		if n < len(f.Data) {
			done = true
		}

		return k, done, nil
	}

	var cur uint64
	var done bool
	for i, bs := range f.GetBlocksizes() {
		if cur+bs > offset {
			child, err := node.Links[i].GetNode(dm.dagserv)
			if err != nil {
				return "", false, err
			}
			k, sdone, err := dm.modifyDag(child, offset-cur, data)
			if err != nil {
				return "", false, err
			}

			offset += bs
			node.Links[i].Hash = mh.Multihash(k)

			if sdone {
				done = true
				break
			}
		}
		cur += bs
	}

	k, err := dm.dagserv.Add(node)
	return k, done, err
}

// appendData appends the blocks from the given chan to the end of this dag
func (dm *DagModifier) appendData(node *mdag.Node, blks <-chan []byte) (*mdag.Node, error) {
	dbp := &help.DagBuilderParams{
		Dagserv:  dm.dagserv,
		Maxlinks: help.DefaultLinksPerBlock,
		Pinner:   dm.mp,
	}

	return trickle.TrickleAppend(node, dbp.New(blks))
}

// Read data from this dag starting at the current offset
func (dm *DagModifier) Read(b []byte) (int, error) {
	err := dm.Flush()
	if err != nil {
		return 0, err
	}

	if dm.read == nil {
		dr, err := uio.NewDagReader(dm.ctx, dm.curNode, dm.dagserv)
		if err != nil {
			return 0, err
		}

		i, err := dr.Seek(int64(dm.curWrOff), os.SEEK_SET)
		if err != nil {
			return 0, err
		}

		if i != int64(dm.curWrOff) {
			return 0, ErrSeekFail
		}

		dm.read = dr
	}

	n, err := dm.read.Read(b)
	dm.curWrOff += uint64(n)
	return n, err
}

// GetNode gets the modified DAG Node
func (dm *DagModifier) GetNode() (*mdag.Node, error) {
	err := dm.Flush()
	if err != nil {
		return nil, err
	}
	return dm.curNode.Copy(), nil
}

// HasChanges returned whether or not there are unflushed changes to this dag
func (dm *DagModifier) HasChanges() bool {
	return dm.wrBuf != nil
}

func (dm *DagModifier) Seek(offset int64, whence int) (int64, error) {
	err := dm.Flush()
	if err != nil {
		return 0, err
	}

	switch whence {
	case os.SEEK_CUR:
		dm.curWrOff += uint64(offset)
		dm.writeStart = dm.curWrOff
	case os.SEEK_SET:
		dm.curWrOff = uint64(offset)
		dm.writeStart = uint64(offset)
	case os.SEEK_END:
		return 0, ErrSeekEndNotImpl
	default:
		return 0, ErrUnrecognizedWhence
	}

	if dm.read != nil {
		_, err = dm.read.Seek(offset, whence)
		if err != nil {
			return 0, err
		}
	}

	return int64(dm.curWrOff), nil
}

func (dm *DagModifier) Truncate(size int64) error {
	err := dm.Flush()
	if err != nil {
		return err
	}

	realSize, err := dm.Size()
	if err != nil {
		return err
	}

	// Truncate can also be used to expand the file
	if size > int64(realSize) {
		return dm.expandSparse(int64(size) - realSize)
	}

	nnode, err := dagTruncate(dm.curNode, uint64(size), dm.dagserv)
	if err != nil {
		return err
	}

	_, err = dm.dagserv.Add(nnode)
	if err != nil {
		return err
	}

	dm.curNode = nnode
	return nil
}

// dagTruncate truncates the given node to 'size' and returns the modified Node
func dagTruncate(nd *mdag.Node, size uint64, ds mdag.DAGService) (*mdag.Node, error) {
	if len(nd.Links) == 0 {
		// TODO: this can likely be done without marshaling and remarshaling
		pbn, err := ft.FromBytes(nd.Data)
		if err != nil {
			return nil, err
		}

		nd.Data = ft.WrapData(pbn.Data[:size])
		return nd, nil
	}

	var cur uint64
	end := 0
	var modified *mdag.Node
	ndata := new(ft.FSNode)
	for i, lnk := range nd.Links {
		child, err := lnk.GetNode(ds)
		if err != nil {
			return nil, err
		}

		childsize, err := ft.DataSize(child.Data)
		if err != nil {
			return nil, err
		}

		// found the child we want to cut
		if size < cur+childsize {
			nchild, err := dagTruncate(child, size-cur, ds)
			if err != nil {
				return nil, err
			}

			// TODO: sanity check size of truncated block
			ndata.AddBlockSize(size - cur)

			modified = nchild
			end = i
			break
		}
		cur += childsize
		ndata.AddBlockSize(childsize)
	}

	_, err := ds.Add(modified)
	if err != nil {
		return nil, err
	}

	nd.Links = nd.Links[:end]
	err = nd.AddNodeLinkClean("", modified)
	if err != nil {
		return nil, err
	}

	d, err := ndata.GetBytes()
	if err != nil {
		return nil, err
	}

	nd.Data = d

	return nd, nil
}
<commit_msg>remove pointless TODO<commit_after>package mod

import (
	"bytes"
	"errors"
	"io"
	"os"

	proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
	mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"

	chunk "github.com/jbenet/go-ipfs/importer/chunk"
	help "github.com/jbenet/go-ipfs/importer/helpers"
	trickle "github.com/jbenet/go-ipfs/importer/trickle"
	mdag "github.com/jbenet/go-ipfs/merkledag"
	pin "github.com/jbenet/go-ipfs/pin"
	ft "github.com/jbenet/go-ipfs/unixfs"
	uio "github.com/jbenet/go-ipfs/unixfs/io"
	u "github.com/jbenet/go-ipfs/util"
)

var ErrSeekFail = errors.New("failed to seek properly")
var ErrSeekEndNotImpl = errors.New("SEEK_END currently not implemented")
var ErrUnrecognizedWhence = errors.New("unrecognized whence")

// 2MB
var writebufferSize = 1 << 21

var log = u.Logger("dagio")

// DagModifier is the only struct licensed and able to correctly
// perform surgery on a DAG 'file'
// Dear god, please rename this to something more pleasant
type DagModifier struct {
	dagserv mdag.DAGService
	curNode *mdag.Node
	mp      pin.ManualPinner

	splitter   chunk.BlockSplitter
	ctx        context.Context
	readCancel func()

	writeStart uint64
	curWrOff   uint64
	wrBuf      *bytes.Buffer

	read *uio.DagReader
}

func NewDagModifier(ctx context.Context, from *mdag.Node, serv mdag.DAGService, mp pin.ManualPinner, spl chunk.BlockSplitter) (*DagModifier, error) {
	return &DagModifier{
		curNode:  from.Copy(),
		dagserv:  serv,
		splitter: spl,
		ctx:      ctx,
		mp:       mp,
	}, nil
}

// WriteAt will modify a dag file in place
func (dm *DagModifier) WriteAt(b []byte, offset int64) (int, error) {
	// TODO: this is currently VERY inneficient
	// each write that happens at an offset other than the current one causes a
	// flush to disk, and dag rewrite
	if uint64(offset) != dm.curWrOff {
		size, err := dm.Size()
		if err != nil {
			return 0, err
		}
		if offset > size {
			err := dm.expandSparse(offset - size)
			if err != nil {
				return 0, err
			}
		}

		err = dm.Flush()
		if err != nil {
			return 0, err
		}
		dm.writeStart = uint64(offset)
	}

	return dm.Write(b)
}

// A reader that just returns zeros
type zeroReader struct{}

func (zr zeroReader) Read(b []byte) (int, error) {
	for i, _ := range b {
		b[i] = 0
	}
	return len(b), nil
}

// expandSparse grows the file with zero blocks of 4096
// A small blocksize is chosen to aid in deduplication
func (dm *DagModifier) expandSparse(size int64) error {
	spl := chunk.SizeSplitter{4096}
	r := io.LimitReader(zeroReader{}, size)
	blks := spl.Split(r)
	nnode, err := dm.appendData(dm.curNode, blks)
	if err != nil {
		return err
	}
	_, err = dm.dagserv.Add(nnode)
	if err != nil {
		return err
	}
	dm.curNode = nnode
	return nil
}

// Write continues writing to the dag at the current offset
func (dm *DagModifier) Write(b []byte) (int, error) {
	if dm.read != nil {
		dm.read = nil
	}
	if dm.wrBuf == nil {
		dm.wrBuf = new(bytes.Buffer)
	}

	n, err := dm.wrBuf.Write(b)
	if err != nil {
		return n, err
	}
	dm.curWrOff += uint64(n)
	if dm.wrBuf.Len() > writebufferSize {
		err := dm.Flush()
		if err != nil {
			return n, err
		}
	}
	return n, nil
}

func (dm *DagModifier) Size() (int64, error) {
	// TODO: compute size without flushing, should be easy
	err := dm.Flush()
	if err != nil {
		return 0, err
	}

	pbn, err := ft.FromBytes(dm.curNode.Data)
	if err != nil {
		return 0, err
	}

	return int64(pbn.GetFilesize()), nil
}

// Flush writes changes to this dag to disk
func (dm *DagModifier) Flush() error {
	// No buffer? Nothing to do
	if dm.wrBuf == nil {
		return nil
	}

	// If we have an active reader, kill it
	if dm.read != nil {
		dm.read = nil
		dm.readCancel()
	}

	// Number of bytes we're going to write
	buflen := dm.wrBuf.Len()

	// overwrite existing dag nodes
	k, done, err := dm.modifyDag(dm.curNode, dm.writeStart, dm.wrBuf)
	if err != nil {
		return err
	}

	nd, err := dm.dagserv.Get(k)
	if err != nil {
		return err
	}

	dm.curNode = nd

	// need to write past end of current dag
	if !done {
		blks := dm.splitter.Split(dm.wrBuf)
		nd, err = dm.appendData(dm.curNode, blks)
		if err != nil {
			return err
		}

		_, err := dm.dagserv.Add(nd)
		if err != nil {
			return err
		}

		dm.curNode = nd
	}

	dm.writeStart += uint64(buflen)

	dm.wrBuf = nil
	return nil
}

// modifyDag writes the data in 'data' over the data in 'node' starting at 'offset'
// returns the new key of the passed in node and whether or not all the data in the reader
// has been consumed.
func (dm *DagModifier) modifyDag(node *mdag.Node, offset uint64, data io.Reader) (u.Key, bool, error) {
	f, err := ft.FromBytes(node.Data)
	if err != nil {
		return "", false, err
	}

	// If we've reached a leaf node.
	if len(node.Links) == 0 {
		n, err := data.Read(f.Data[offset:])
		if err != nil && err != io.EOF {
			return "", false, err
		}

		// Update newly written node..
		b, err := proto.Marshal(f)
		if err != nil {
			return "", false, err
		}

		nd := &mdag.Node{Data: b}
		k, err := dm.dagserv.Add(nd)
		if err != nil {
			return "", false, err
		}

		// Hey look! we're done!
		var done bool
		if n < len(f.Data) {
			done = true
		}

		return k, done, nil
	}

	var cur uint64
	var done bool
	for i, bs := range f.GetBlocksizes() {
		if cur+bs > offset {
			child, err := node.Links[i].GetNode(dm.dagserv)
			if err != nil {
				return "", false, err
			}
			k, sdone, err := dm.modifyDag(child, offset-cur, data)
			if err != nil {
				return "", false, err
			}

			offset += bs
			node.Links[i].Hash = mh.Multihash(k)

			if sdone {
				done = true
				break
			}
		}
		cur += bs
	}

	k, err := dm.dagserv.Add(node)
	return k, done, err
}

// appendData appends the blocks from the given chan to the end of this dag
func (dm *DagModifier) appendData(node *mdag.Node, blks <-chan []byte) (*mdag.Node, error) {
	dbp := &help.DagBuilderParams{
		Dagserv:  dm.dagserv,
		Maxlinks: help.DefaultLinksPerBlock,
		Pinner:   dm.mp,
	}

	return trickle.TrickleAppend(node, dbp.New(blks))
}

// Read data from this dag starting at the current offset
func (dm *DagModifier) Read(b []byte) (int, error) {
	err := dm.Flush()
	if err != nil {
		return 0, err
	}

	if dm.read == nil {
		dr, err := uio.NewDagReader(dm.ctx, dm.curNode, dm.dagserv)
		if err != nil {
			return 0, err
		}

		i, err := dr.Seek(int64(dm.curWrOff), os.SEEK_SET)
		if err != nil {
			return 0, err
		}

		if i != int64(dm.curWrOff) {
			return 0, ErrSeekFail
		}

		dm.read = dr
	}

	n, err := dm.read.Read(b)
	dm.curWrOff += uint64(n)
	return n, err
}

// GetNode gets the modified DAG Node
func (dm *DagModifier) GetNode() (*mdag.Node, error) {
	err := dm.Flush()
	if err != nil {
		return nil, err
	}
	return dm.curNode.Copy(), nil
}

// HasChanges returned whether or not there are unflushed changes to this dag
func (dm *DagModifier) HasChanges() bool {
	return dm.wrBuf != nil
}

func (dm *DagModifier) Seek(offset int64, whence int) (int64, error) {
	err := dm.Flush()
	if err != nil {
		return 0, err
	}

	switch whence {
	case os.SEEK_CUR:
		dm.curWrOff += uint64(offset)
		dm.writeStart = dm.curWrOff
	case os.SEEK_SET:
		dm.curWrOff = uint64(offset)
		dm.writeStart = uint64(offset)
	case os.SEEK_END:
		return 0, ErrSeekEndNotImpl
	default:
		return 0, ErrUnrecognizedWhence
	}

	if dm.read != nil {
		_, err = dm.read.Seek(offset, whence)
		if err != nil {
			return 0, err
		}
	}

	return int64(dm.curWrOff), nil
}

func (dm *DagModifier) Truncate(size int64) error {
	err := dm.Flush()
	if err != nil {
		return err
	}

	realSize, err := dm.Size()
	if err != nil {
		return err
	}

	// Truncate can also be used to expand the file
	if size > int64(realSize) {
		return dm.expandSparse(int64(size) - realSize)
	}

	nnode, err := dagTruncate(dm.curNode, uint64(size), dm.dagserv)
	if err != nil {
		return err
	}

	_, err = dm.dagserv.Add(nnode)
	if err != nil {
		return err
	}

	dm.curNode = nnode
	return nil
}

// dagTruncate truncates the given node to 'size' and returns the modified Node
func dagTruncate(nd *mdag.Node, size uint64, ds mdag.DAGService) (*mdag.Node, error) {
	if len(nd.Links) == 0 {
		// TODO: this can likely be done without marshaling and remarshaling
		pbn, err := ft.FromBytes(nd.Data)
		if err != nil {
			return nil, err
		}

		nd.Data = ft.WrapData(pbn.Data[:size])
		return nd, nil
	}

	var cur uint64
	end := 0
	var modified *mdag.Node
	ndata := new(ft.FSNode)
	for i, lnk := range nd.Links {
		child, err := lnk.GetNode(ds)
		if err != nil {
			return nil, err
		}

		childsize, err := ft.DataSize(child.Data)
		if err != nil {
			return nil, err
		}

		// found the child we want to cut
		if size < cur+childsize {
			nchild, err := dagTruncate(child, size-cur, ds)
			if err != nil {
				return nil, err
			}

			ndata.AddBlockSize(size - cur)

			modified = nchild
			end = i
			break
		}
		cur += childsize
		ndata.AddBlockSize(childsize)
	}

	_, err := ds.Add(modified)
	if err != nil {
		return nil, err
	}

	nd.Links = nd.Links[:end]
	err = nd.AddNodeLinkClean("", modified)
	if err != nil {
		return nil, err
	}

	d, err := ndata.GetBytes()
	if err != nil {
		return nil, err
	}

	nd.Data = d

	return nd, nil
}
<|endoftext|>


<commit_before>package nsq

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"strconv"
	"strings"
)

type Command struct {
	Name   []byte
	Params [][]byte
	Body   []byte
}

func (c *Command) String() string {
	if len(c.Params) > 0 {
		return fmt.Sprintf("%s %s", c.Name, string(bytes.Join(c.Params, []byte(" "))))
	}
	return string(c.Name)
}

// Announce creates a new Command to announce the existence of
// a given topic and/or channel.
// NOTE: if channel == "." then it is considered n/a
func Announce(topic string, channel string, port int, ips []string) *Command {
	var params = [][]byte{[]byte(topic), []byte(channel), []byte(strconv.Itoa(port))}
	return &Command{[]byte("ANNOUNCE"), params, []byte(strings.Join(ips, "\n"))}
}

// Identify is the first message sent to the Lookupd and provides information about the client
func Identify(version string, tcpPort int, httpPort int, address string) *Command {
	body, err := json.Marshal(struct {
		Version  string `json:"version"`
		TcpPort  int    `json:"tcp_port"`
		HttpPort int    `json:"http_port"`
		Address  string `json:"address"`
	}{
		version,
		tcpPort,
		httpPort,
		address,
	})
	if err != nil {
		log.Fatal("failed to create json %s", err.Error())
	}
	return &Command{[]byte("IDENTIFY"), [][]byte{}, body}
}

// REGISTER a topic/channel for this nsqd
func Register(topic string, channel string) *Command {
	params := [][]byte{[]byte(topic)}
	if len(channel) > 0 {
		params = append(params, []byte(channel))
	}
	return &Command{[]byte("REGISTER"), params, nil}
}

// UNREGISTER removes a topic/channel from this nsqd
func UnRegister(topic string, channel string) *Command {
	params := [][]byte{[]byte(topic)}
	if len(channel) > 0 {
		params = append(params, []byte(channel))
	}
	return &Command{[]byte("UNREGISTER"), params, nil}
}

// Ping creates a new Command to keep-alive the state of all the
// announced topic/channels for a given client
func Ping() *Command {
	return &Command{[]byte("PING"), nil, nil}
}

// Publish creates a new Command to write a message to a given topic
func Publish(topic string, body []byte) *Command {
	var params = [][]byte{[]byte(topic)}
	return &Command{[]byte("PUB"), params, body}
}

// Subscribe creates a new Command to subscribe
// to the given topic/channel
func Subscribe(topic string, channel string, shortIdentifier string, longIdentifier string) *Command {
	var params = [][]byte{[]byte(topic), []byte(channel), []byte(shortIdentifier), []byte(longIdentifier)}
	return &Command{[]byte("SUB"), params, nil}
}

// Ready creates a new Command to specify
// the number of messages a client is willing to receive
func Ready(count int) *Command {
	var params = [][]byte{[]byte(strconv.Itoa(count))}
	return &Command{[]byte("RDY"), params, nil}
}

// Finish creates a new Command to indiciate that
// a given message (by id) has been processed successfully
func Finish(id []byte) *Command {
	var params = [][]byte{id}
	return &Command{[]byte("FIN"), params, nil}
}

// Requeue creats a new Command to indicate that
// a given message (by id) should be requeued after the given timeout (in ms)
// NOTE: a timeout of 0 indicates immediate requeue
func Requeue(id []byte, timeoutMs int) *Command {
	var params = [][]byte{id, []byte(strconv.Itoa(timeoutMs))}
	return &Command{[]byte("REQ"), params, nil}
}

// StartClose creates a new Command to indicate that the
// client would like to start a close cycle.  nsqd will no longer
// send messages to a client in this state and the client is expected
// to ACK after which it can finish pending messages and close the connection
func StartClose() *Command {
	return &Command{[]byte("CLS"), nil, nil}
}

func Nop() *Command {
	return &Command{[]byte("NOP"), nil, nil}
}
<commit_msg>nsqd: add MPUB (MultiPublish) protocol command<commit_after>package nsq

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"strconv"
	"strings"
)

type Command struct {
	Name   []byte
	Params [][]byte
	Body   []byte
}

func (c *Command) String() string {
	if len(c.Params) > 0 {
		return fmt.Sprintf("%s %s", c.Name, string(bytes.Join(c.Params, []byte(" "))))
	}
	return string(c.Name)
}

// Announce creates a new Command to announce the existence of
// a given topic and/or channel.
// NOTE: if channel == "." then it is considered n/a
func Announce(topic string, channel string, port int, ips []string) *Command {
	var params = [][]byte{[]byte(topic), []byte(channel), []byte(strconv.Itoa(port))}
	return &Command{[]byte("ANNOUNCE"), params, []byte(strings.Join(ips, "\n"))}
}

// Identify is the first message sent to the Lookupd and provides information about the client
func Identify(version string, tcpPort int, httpPort int, address string) *Command {
	body, err := json.Marshal(struct {
		Version  string `json:"version"`
		TcpPort  int    `json:"tcp_port"`
		HttpPort int    `json:"http_port"`
		Address  string `json:"address"`
	}{
		version,
		tcpPort,
		httpPort,
		address,
	})
	if err != nil {
		log.Fatal("failed to create json %s", err.Error())
	}
	return &Command{[]byte("IDENTIFY"), [][]byte{}, body}
}

// REGISTER a topic/channel for this nsqd
func Register(topic string, channel string) *Command {
	params := [][]byte{[]byte(topic)}
	if len(channel) > 0 {
		params = append(params, []byte(channel))
	}
	return &Command{[]byte("REGISTER"), params, nil}
}

// UNREGISTER removes a topic/channel from this nsqd
func UnRegister(topic string, channel string) *Command {
	params := [][]byte{[]byte(topic)}
	if len(channel) > 0 {
		params = append(params, []byte(channel))
	}
	return &Command{[]byte("UNREGISTER"), params, nil}
}

// Ping creates a new Command to keep-alive the state of all the
// announced topic/channels for a given client
func Ping() *Command {
	return &Command{[]byte("PING"), nil, nil}
}

// Publish creates a new Command to write a message to a given topic
func Publish(topic string, body []byte) *Command {
	var params = [][]byte{[]byte(topic)}
	return &Command{[]byte("PUB"), params, body}
}

func MultiPublish(topic string, bodies [][]byte) (*Command, error) {
	var params = [][]byte{[]byte(topic)}

	num := uint32(len(bodies))
	bodySize := 4
	for _, b := range bodies {
		bodySize += len(b) + 4
	}
	body := make([]byte, bodySize)
	buf := bytes.NewBuffer(body)

	err := binary.Write(buf, binary.BigEndian, &num)
	if err != nil {
		return nil, err
	}
	for _, b := range bodies {
		err = binary.Write(buf, binary.BigEndian, int32(len(b)))
		if err != nil {
			return nil, err
		}
		_, err = buf.Write(b)
		if err != nil {
			return nil, err
		}
	}

	return &Command{[]byte("MPUB"), params, body}, nil
}

// Subscribe creates a new Command to subscribe
// to the given topic/channel
func Subscribe(topic string, channel string, shortIdentifier string, longIdentifier string) *Command {
	var params = [][]byte{[]byte(topic), []byte(channel), []byte(shortIdentifier), []byte(longIdentifier)}
	return &Command{[]byte("SUB"), params, nil}
}

// Ready creates a new Command to specify
// the number of messages a client is willing to receive
func Ready(count int) *Command {
	var params = [][]byte{[]byte(strconv.Itoa(count))}
	return &Command{[]byte("RDY"), params, nil}
}

// Finish creates a new Command to indiciate that
// a given message (by id) has been processed successfully
func Finish(id []byte) *Command {
	var params = [][]byte{id}
	return &Command{[]byte("FIN"), params, nil}
}

// Requeue creats a new Command to indicate that
// a given message (by id) should be requeued after the given timeout (in ms)
// NOTE: a timeout of 0 indicates immediate requeue
func Requeue(id []byte, timeoutMs int) *Command {
	var params = [][]byte{id, []byte(strconv.Itoa(timeoutMs))}
	return &Command{[]byte("REQ"), params, nil}
}

// StartClose creates a new Command to indicate that the
// client would like to start a close cycle.  nsqd will no longer
// send messages to a client in this state and the client is expected
// to ACK after which it can finish pending messages and close the connection
func StartClose() *Command {
	return &Command{[]byte("CLS"), nil, nil}
}

func Nop() *Command {
	return &Command{[]byte("NOP"), nil, nil}
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package upgrades_test

import (
	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/testing/testbase"
	"launchpad.net/juju-core/upgrades"
)

type steps118Suite struct {
	testbase.LoggingSuite
}

var _ = gc.Suite(&steps118Suite{})

var expectedSteps = []string{
	"make $DATADIR/locks owned by ubuntu:ubuntu",
	"generate system ssh key",
	"update rsyslog port",
	"install rsyslog-gnutls",
	"remove deprecated environment config settings",
	"migrate local provider agent config",
}

func (s *steps118Suite) TestUpgradeOperationsContent(c *gc.C) {
	upgradeSteps := upgrades.StepsFor118()
	c.Assert(upgradeSteps, gc.HasLen, len(expectedSteps))
	assertExpectedSteps(c, upgradeSteps, expectedSteps)
}
<commit_msg>Add the extra step to the 1.18 test.<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package upgrades_test

import (
	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/testing/testbase"
	"launchpad.net/juju-core/upgrades"
)

type steps118Suite struct {
	testbase.LoggingSuite
}

var _ = gc.Suite(&steps118Suite{})

var expectedSteps = []string{
	"make $DATADIR/locks owned by ubuntu:ubuntu",
	"generate system ssh key",
	"update rsyslog port",
	"install rsyslog-gnutls",
	"remove deprecated environment config settings",
	"migrate local provider agent config",
	"make /home/ubuntu/.profile source .juju-proxy file",
}

func (s *steps118Suite) TestUpgradeOperationsContent(c *gc.C) {
	upgradeSteps := upgrades.StepsFor118()
	c.Assert(upgradeSteps, gc.HasLen, len(expectedSteps))
	assertExpectedSteps(c, upgradeSteps, expectedSteps)
}
<|endoftext|>


<commit_before>package main

import "github.com/codegangsta/cli"

// Commands
var Commands = []cli.Command{
	commandDevices,
	commandPlay,
}

var commandDevices = cli.Command{
	Name:  "devices",
	Usage: "Show AirPlay devices",
	Description: `
`,
	Action: Devices,
}

var commandPlay = cli.Command{
	Name:  "play",
	Usage: "Play media file",
	Description: `
`,
	Action: Play,
}
<commit_msg>Simplify define commands<commit_after>package main

import (
	"fmt"

	"github.com/codegangsta/cli"
	"github.com/gongo/go-airplay"
)

// Define commands
var Commands = []cli.Command{
	{
		Name:  "play",
		Usage: "Play media file",
		Action: func(c *cli.Context) {
			client := airplay.NewClient()
			ch := client.Play(source(c.Args().First()))
			<-ch
		},
	},
	{
		Name:  "devices",
		Usage: "Show AirPlay devices",
		Action: func(c *cli.Context) {
			for _, d := range Devices() {
				fmt.Println(d.Name)
			}
		},
	},
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package codegen_test

import (
	"os"
	"path/filepath"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/uber/zanzibar/codegen"
)

var fooThrift = filepath.Join(
	os.Getenv("GOPATH"),
	"/src/github.com/uber/zanzibar/",
	"examples/example-gateway/idl/",
	"github.com/uber/zanzibar/clients/foo/foo.thrift")

func newPackageHelper(t *testing.T) *codegen.PackageHelper {
	relativeGatewayPath := "../../examples/example-gateway"
	absGatewayPath, err := filepath.Abs(relativeGatewayPath)
	if !assert.NoError(t, err, "failed to get abs path %s", err) {
		return nil
	}

	h, err := codegen.NewPackageHelper(
		"examples/example-gateway/idl",
		"examples/example-gateway/gen-code",
		tmpDir,
		filepath.Join(absGatewayPath, "idl/github.com/uber/zanzibar"),
	)
	if !assert.NoError(t, err, "failed to create package helper") {
		return nil
	}
	return h
}

func TestImportPath(t *testing.T) {
	h := newPackageHelper(t)
	p, err := h.TypeImportPath(fooThrift)
	assert.Nil(t, err, "should not return error")
	assert.Equal(t, "github.com/uber/zanzibar/examples/example-gateway/build/gen-code/github.com/uber/zanzibar/clients/foo/foo", p, "wrong type import path")
	_, err = h.TypeImportPath("/Users/xxx/go/src/github.com/uber/zanzibar/examples/example-gateway/build/idl/github.com/uber/zanzibar/clients/foo/foo.go")
	assert.Error(t, err, "should return error for not a thrift file")
	_, err = h.TypeImportPath("/Users/xxx/go/src/github.com/uber/zanzibar/examples/example-gateway/build/zanzibar/clients/foo/foo.thrift")
	assert.Error(t, err, "should return error for not in IDL dir")
}

func TestTypePackageName(t *testing.T) {
	h := newPackageHelper(t)
	packageName, err := h.TypePackageName(fooThrift)
	assert.Nil(t, err, "should not return error")
	assert.Equal(t, "foo", packageName, "wrong package name")
	_, err = h.TypeImportPath("/Users/xxx/go/src/github.com/uber/zanzibar/examples/example-gateway/build/idl/github.com/uber/zanzibar/clients/foo/foo.txt")
	assert.Error(t, err, "should return error for not a thrift file")
}

func TestPackageGenPath(t *testing.T) {
	h := newPackageHelper(t)
	p, err := h.PackageGenPath(fooThrift)
	assert.Nil(t, err, "should not return error")
	exp := "github.com/uber/zanzibar/.tmp_gen/clients/foo"
	assert.Equal(t, exp, p, "wrong generated Go package path")
	_, err = h.PackageGenPath("/Users/xxx/go/src/github.com/uber/zanzibar/examples/example-gateway/build/idl/github.com/uber/zanzibar/clients/foo/foo.go")
	assert.Error(t, err, "should return error for not a thrift file")
	_, err = h.PackageGenPath("/Users/xxx/go/src/github.com/uber/zanzibar/examples/example-gateway/build/zanzibar/clients/foo/foo.thrift")
	assert.Error(t, err, "should return error for not in IDL dir")
}

func TestEndpointTestConfigPath(t *testing.T) {
	h := newPackageHelper(t)
	p := h.EndpointTestConfigPath("foo", "bar")
	exp := "foo/bar_test.json"
	assert.Equal(t, exp, p, "wrong generated endpoint test config path")
}
<commit_msg>fix package_test ???<commit_after>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package codegen_test

import (
	"os"
	"path/filepath"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/uber/zanzibar/codegen"
)

var barThrift = filepath.Join(
	os.Getenv("GOPATH"),
	"/src/github.com/uber/zanzibar/",
	"examples/example-gateway/idl/",
	"github.com/uber/zanzibar/clients/bar/bar.thrift")

func newPackageHelper(t *testing.T) *codegen.PackageHelper {
	relativeGatewayPath := "../../examples/example-gateway"
	absGatewayPath, err := filepath.Abs(relativeGatewayPath)
	if !assert.NoError(t, err, "failed to get abs path %s", err) {
		return nil
	}

	h, err := codegen.NewPackageHelper(
		filepath.Join(absGatewayPath, "idl"),
		"github.com/uber/zanzibar/examples/example-gateway/build/gen-code",
		tmpDir,
		filepath.Join(absGatewayPath, "idl/github.com/uber/zanzibar"),
	)
	if !assert.NoError(t, err, "failed to create package helper") {
		return nil
	}
	return h
}

func TestImportPath(t *testing.T) {
	h := newPackageHelper(t)
	p, err := h.TypeImportPath(barThrift)
	assert.Nil(t, err, "should not return error")
	assert.Equal(t, "github.com/uber/zanzibar/examples/example-gateway/build/gen-code/github.com/uber/zanzibar/clients/bar/bar", p, "wrong type import path")
	_, err = h.TypeImportPath("/Users/xxx/go/src/github.com/uber/zanzibar/examples/example-gateway/build/idl/github.com/uber/zanzibar/clients/bar/bar.go")
	assert.Error(t, err, "should return error for not a thrift file")
	_, err = h.TypeImportPath("/Users/xxx/go/src/github.com/uber/zanzibar/examples/example-gateway/build/zanzibar/clients/bar/bar.thrift")
	assert.Error(t, err, "should return error for not in IDL dir")
}

func TestTypePackageName(t *testing.T) {
	h := newPackageHelper(t)
	packageName, err := h.TypePackageName(barThrift)
	assert.Nil(t, err, "should not return error")
	assert.Equal(t, "bar", packageName, "wrong package name")
	_, err = h.TypeImportPath("/Users/xxx/go/src/github.com/uber/zanzibar/examples/example-gateway/build/idl/github.com/uber/zanzibar/clients/bar/bar.txt")
	assert.Error(t, err, "should return error for not a thrift file")
}

func TestPackageGenPath(t *testing.T) {
	h := newPackageHelper(t)
	p, err := h.PackageGenPath(barThrift)
	assert.Nil(t, err, "should not return error")
	exp := "github.com/uber/zanzibar/.tmp_gen/clients/bar"
	assert.Equal(t, exp, p, "wrong generated Go package path")
	_, err = h.PackageGenPath("/Users/xxx/go/src/github.com/uber/zanzibar/examples/example-gateway/build/idl/github.com/uber/zanzibar/clients/bar/bar.go")
	assert.Error(t, err, "should return error for not a thrift file")
	_, err = h.PackageGenPath("/Users/xxx/go/src/github.com/uber/zanzibar/examples/example-gateway/build/zanzibar/clients/bar/bar.thrift")
	assert.Error(t, err, "should return error for not in IDL dir")
}

func TestEndpointTestConfigPath(t *testing.T) {
	h := newPackageHelper(t)
	p := h.EndpointTestConfigPath("bar", "bar")
	exp := "bar/bar_test.json"
	assert.Equal(t, exp, p, "wrong generated endpoint test config path")
}
<|endoftext|>


<commit_before>// Package jsql provides glue between database/sql and json.
package jsql

import (
	"bytes"
	"database/sql"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"mime"
	"net/http"
	"regexp"

	"github.com/gorilla/mux"
)

const jsonContentType = "application/json;charset=UTF-8"

var reVars = regexp.MustCompile(`\${([^}]+)}`)

// PositionalQueryVars controls wether
// the generated query strings will use '?'
// or $1, $2... syntax.
var PositionalQueryVars = true

// TODO(lvd) less simplistic parsing of query.  worry about quoting etc.
func rewrite(q string) (qq string, varnames []string) {
	idx := reVars.FindAllStringSubmatchIndex(q, -1)
	l := 0
	var b bytes.Buffer
	for _, v := range idx {
		varnames = append(varnames, q[v[2]:v[3]])
		b.WriteString(q[l:v[0]])
		l = v[1]
		if PositionalQueryVars {
			b.WriteString(fmt.Sprintf("$%d", len(varnames)))
		} else {
			b.WriteString("?")
		}
	}
	b.WriteString(q[l:])
	return b.String(), varnames
}

// TODO:make Q send records on a channel, and provide json/bson as encoders
// then have the Handler switch returned encoding depending on http headers

// A queryfunc is the type of function generated by Q.
// It executes a query substituting arguments from the provided args map,
// and writes a JSON array of objects to the provided writer.
// It returns the number of array elements written and any error that may have occured.
//
// If w is an http.ResponseWriter (has a Header() http.Header method), then it will set
// the Content-Type header to application/json;charset=UTF-8 before the first write.
//
// If n == 0, nothing, not even the opening bracket, will have been written to w,
// and the header will not have been set, meaning it is not too late to call e.g.
//
//    http.Error(w, err.Error(), http.StatusInternalError)
//
// otherwise the error will have to be tacked on to the already produced output or
// reported out of band.  If any elements have been written the queryfunc will
// always generate a proper closing bracket.
type QueryFunc func(args map[string]interface{}, w io.Writer) (n int, err error)

// Q builds a function that can execute the query on a database given
// a map of parameters, and writes the result as json to a writer.
//
// The query must be of the form 'SELECT foo FROM bar WHERE x = ${varname}'
// that is, any query accepted by the driver that will provide db,
// but with the normal '$1, $2...' parameters replace by ${varname}.
//
// The generated function will extract the corresponding values from the args map
// and supply them to the db.Stmt.Execute method in the correct order.
// The query will be rewritten to use $1, $2 .. etc.
//
// The generated json will be an array of objects, where
// each object has a name taken from the columns as named in the SQL query.
//
func Q(db *sql.DB, query string) (QueryFunc, error) {
	q, argn := rewrite(query)
	stmt, err := db.Prepare(q)
	if err != nil {
		return nil, err
	}
	return func(args map[string]interface{}, w io.Writer) (n int, err error) {
		var argv []interface{}
		for _, v := range argn {
			argv = append(argv, args[v])
		}
		rows, err := stmt.Query(argv...)
		if err != nil {
			return 0, err
		}
		defer rows.Close()
		retn, err := rows.Columns()
		if err != nil {
			return 0, err
		}

		var (
			retv  = make([]interface{}, len(retn))
			retvv = make([]interface{}, len(retn))
			retm  = make(map[string]interface{})
		)
		for i := range retv {
			retvv[i] = &retv[i]
			retm[retn[i]] = &retv[i]
		}
		for rows.Next() {
			if err := rows.Scan(retvv...); err != nil {
				return n, err
			}
			// convert []byte to strings, because json uuencodes []byte
			for i, v := range retv {
				if vv, ok := v.([]byte); ok {
					retv[i] = string(vv)
				}
			}

			b, err := json.Marshal(retm)
			if err != nil {
				return n, err
			}

			if n == 0 {
				if rw, ok := w.(http.ResponseWriter); ok {
					rw.Header().Set("Content-type", jsonContentType)
				}
				w.Write([]byte("[\n"))
				defer w.Write([]byte("\n]"))
			} else {
				w.Write([]byte(",\n"))
			}

			w.Write(b)
			n++
		}

		return n, rows.Err()
	}, nil
}

// Handler is a convenience wrapper around MkHandler
// that will die on error.
func Handler(db *sql.DB, query string) http.Handler {
	h, err := MkHandler(db, query)
	if err != nil {
		log.Fatal(err)
	}
	return h
}

// MkHandler produces a http.Handler that takes arguments from
// the request, and runs Q, producing a nice error message if needed.
//
// The arguments are taken from the request's formvalues or any
// input json object, depending on the requests content type, merged with
// the "github.com/gorilla/mux".Vars(r), which take precedence
// in case of name conflicts.
func MkHandler(db *sql.DB, query string) (http.Handler, error) {
	_, names := rewrite(query)

	qf, err := Q(db, query)
	if err != nil {
		return nil, err
	}

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		args := make(map[string]interface{})

		var jsonargs map[string]interface{}
		if r.Method == "POST" || r.Method == "PUT" {
			ct := r.Header.Get("Content-Type")
			if ct == "" {
				ct = "application/octet-stream"
			}
			ct, _, err = mime.ParseMediaType(ct)
			if ct == "application/json" {
				defer r.Body.Close()
				if err := json.NewDecoder(io.LimitReader(r.Body, 128<<10)).Decode(&jsonargs); err != nil {
					if err != io.EOF { // empty body is allowed
						http.Error(w, fmt.Sprintf("Can't decode json request: %v", err), http.StatusBadRequest)
						return
					}
				}
			}
		}
		muxargs := mux.Vars(r)
		// TODO harden against malicious input
		// for now we just rely on precedence: muxargs trum the others
		// but this still allows arbitrary values to end up in the query parameters
		for _, n := range names {
			if v, ok := muxargs[n]; ok {
				args[n] = v
				continue
			}
			if v := r.FormValue(n); v != "" {
				args[n] = v
				continue
			}
			if v, ok := jsonargs[n]; ok {
				args[n] = v
				continue
			}
		}
		// TBD: what if not all names set? can postgres $n deal with nil?
		n, err := qf(args, w)
		if err != nil {
			if n == 0 {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			log.Print(r.Method, r.URL, ": ", err)
		}
	}), nil
}
<commit_msg>clarified comment<commit_after>// Package jsql provides glue between database/sql and json.
package jsql

import (
	"bytes"
	"database/sql"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"mime"
	"net/http"
	"regexp"

	"github.com/gorilla/mux"
)

const jsonContentType = "application/json;charset=UTF-8"

var reVars = regexp.MustCompile(`\${([^}]+)}`)

// PositionalQueryVars controls wether
// the generated query strings will use '?'
// or $1, $2... syntax.
var PositionalQueryVars = true

// TODO(lvd) less simplistic parsing of query.  worry about quoting etc.
func rewrite(q string) (qq string, varnames []string) {
	idx := reVars.FindAllStringSubmatchIndex(q, -1)
	l := 0
	var b bytes.Buffer
	for _, v := range idx {
		varnames = append(varnames, q[v[2]:v[3]])
		b.WriteString(q[l:v[0]])
		l = v[1]
		if PositionalQueryVars {
			b.WriteString(fmt.Sprintf("$%d", len(varnames)))
		} else {
			b.WriteString("?")
		}
	}
	b.WriteString(q[l:])
	return b.String(), varnames
}

// TODO:make Q send records on a channel, and provide json/bson as encoders
// then have the Handler switch returned encoding depending on http headers

// A queryfunc is the type of function generated by Q.
// It executes a query substituting arguments from the provided args map,
// and writes a JSON array of objects to the provided writer.
// It returns the number of array elements written and any error that may have occured.
//
// If w is an http.ResponseWriter (has a Header() http.Header method), then it will set
// the Content-Type header to application/json;charset=UTF-8 before the first write.
//
// If n == 0, nothing, not even the opening bracket, will have been written to w,
// and the header will not have been set, meaning it is not too late to call e.g.
//
//    http.Error(w, err.Error(), http.StatusInternalError)
//
// otherwise the error will have to be tacked on to the already produced output or
// reported out of band.  If any elements have been written the queryfunc will
// always generate a proper closing bracket.
type QueryFunc func(args map[string]interface{}, w io.Writer) (n int, err error)

// Q builds a function that can execute the query on a database given
// a map of parameters, and writes the result as json to a writer.
//
// The query must be of the form 'SELECT foo FROM bar WHERE x = ${varname}'
// that is, any query accepted by the driver that will provide db,
// but with the normal '$1, $2...' parameters replace by ${varname}.
//
// The generated function will extract the corresponding values from the args map
// and supply them to the db.Stmt.Execute method in the correct order.
// The query will be rewritten to use $1, $2 .. etc.
//
// The generated json will be an array of objects, one per row where
// each row object has keys named after the columns in the SQL query.
//
func Q(db *sql.DB, query string) (QueryFunc, error) {
	q, argn := rewrite(query)
	stmt, err := db.Prepare(q)
	if err != nil {
		return nil, err
	}
	return func(args map[string]interface{}, w io.Writer) (n int, err error) {
		var argv []interface{}
		for _, v := range argn {
			argv = append(argv, args[v])
		}
		rows, err := stmt.Query(argv...)
		if err != nil {
			return 0, err
		}
		defer rows.Close()
		retn, err := rows.Columns()
		if err != nil {
			return 0, err
		}

		var (
			retv  = make([]interface{}, len(retn))
			retvv = make([]interface{}, len(retn))
			retm  = make(map[string]interface{})
		)
		for i := range retv {
			retvv[i] = &retv[i]
			retm[retn[i]] = &retv[i]
		}
		for rows.Next() {
			if err := rows.Scan(retvv...); err != nil {
				return n, err
			}
			// convert []byte to strings, because json uuencodes []byte
			for i, v := range retv {
				if vv, ok := v.([]byte); ok {
					retv[i] = string(vv)
				}
			}

			b, err := json.Marshal(retm)
			if err != nil {
				return n, err
			}

			if n == 0 {
				if rw, ok := w.(http.ResponseWriter); ok {
					rw.Header().Set("Content-type", jsonContentType)
				}
				w.Write([]byte("[\n"))
				defer w.Write([]byte("\n]"))
			} else {
				w.Write([]byte(",\n"))
			}

			w.Write(b)
			n++
		}

		return n, rows.Err()
	}, nil
}

// Handler is a convenience wrapper around MkHandler
// that will die on error.
func Handler(db *sql.DB, query string) http.Handler {
	h, err := MkHandler(db, query)
	if err != nil {
		log.Fatal(err)
	}
	return h
}

// MkHandler produces a http.Handler that takes arguments from
// the request, and runs Q, producing a nice error message if needed.
//
// The arguments are taken from the request's formvalues or any
// input json object, depending on the requests content type, merged with
// the "github.com/gorilla/mux".Vars(r), which take precedence
// in case of name conflicts.
func MkHandler(db *sql.DB, query string) (http.Handler, error) {
	_, names := rewrite(query)

	qf, err := Q(db, query)
	if err != nil {
		return nil, err
	}

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		args := make(map[string]interface{})

		var jsonargs map[string]interface{}
		if r.Method == "POST" || r.Method == "PUT" {
			ct := r.Header.Get("Content-Type")
			if ct == "" {
				ct = "application/octet-stream"
			}
			ct, _, err = mime.ParseMediaType(ct)
			if ct == "application/json" {
				defer r.Body.Close()
				if err := json.NewDecoder(io.LimitReader(r.Body, 128<<10)).Decode(&jsonargs); err != nil {
					if err != io.EOF { // empty body is allowed
						http.Error(w, fmt.Sprintf("Can't decode json request: %v", err), http.StatusBadRequest)
						return
					}
				}
			}
		}
		muxargs := mux.Vars(r)
		// TODO harden against malicious input
		// for now we just rely on precedence: muxargs trum the others
		// but this still allows arbitrary values to end up in the query parameters
		for _, n := range names {
			if v, ok := muxargs[n]; ok {
				args[n] = v
				continue
			}
			if v := r.FormValue(n); v != "" {
				args[n] = v
				continue
			}
			if v, ok := jsonargs[n]; ok {
				args[n] = v
				continue
			}
		}
		// TBD: what if not all names set? can postgres $n deal with nil?
		n, err := qf(args, w)
		if err != nil {
			if n == 0 {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			log.Print(r.Method, r.URL, ": ", err)
		}
	}), nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"syscall"
	"unsafe"
)

/*** defines ***/

const KILO_VERSION = "0.0.1"
const (
	ARROW_LEFT  = 1000 + iota
	ARROW_RIGHT = 1000 + iota
	ARROW_UP    = 1000 + iota
	ARROW_DOWN  = 1000 + iota
	DEL_KEY     = 1000 + iota
	HOME_KEY    = 1000 + iota
	END_KEY     = 1000 + iota
	PAGE_UP     = 1000 + iota
	PAGE_DOWN   = 1000 + iota
)

/*** data ***/

type Termios struct {
	Iflag  uint32
	Oflag  uint32
	Cflag  uint32
	Lflag  uint32
	Cc     [20]byte
	Ispeed uint32
	Ospeed uint32
}

type erow struct {
	chars []byte
}

type editorConfig struct {
	cx          int
	cy          int
	rowoff      int
	coloff      int
	screenRows  int
	screenCols  int
	numRows     int
	rows        []erow
	origTermios *Termios
}

type WinSize struct {
	Row    uint16
	Col    uint16
	Xpixel uint16
	Ypixel uint16
}

var E editorConfig

/*** terminal ***/

func die(err error) {
	disableRawMode()
	io.WriteString(os.Stdout, "\x1b[2J")
	io.WriteString(os.Stdout, "\x1b[H")
	log.Fatal(err)
}

func TcSetAttr(fd uintptr, termios *Termios) error {
	// TCSETS+1 == TCSETSW, because TCSAFLUSH doesn't exist
	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, uintptr(syscall.TCSETS+1), uintptr(unsafe.Pointer(termios))); err != 0 {
		return err
	}
	return nil
}

func TcGetAttr(fd uintptr) *Termios {
	var termios = &Termios{}
	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, syscall.TCGETS, uintptr(unsafe.Pointer(termios))); err != 0 {
		log.Fatalf("Problem getting terminal attributes: %s\n", err)
	}
	return termios
}

func enableRawMode() {
	E.origTermios = TcGetAttr(os.Stdin.Fd())
	var raw Termios
	raw = *E.origTermios
	raw.Iflag &^= syscall.BRKINT | syscall.ICRNL | syscall.INPCK | syscall.ISTRIP | syscall.IXON
	raw.Oflag &^= syscall.OPOST
	raw.Cflag |= syscall.CS8
	raw.Lflag &^= syscall.ECHO | syscall.ICANON | syscall.IEXTEN | syscall.ISIG
	raw.Cc[syscall.VMIN+1] = 0
	raw.Cc[syscall.VTIME+1] = 1
	if e := TcSetAttr(os.Stdin.Fd(), &raw); e != nil {
		log.Fatalf("Problem enabling raw mode: %s\n", e)
	}
}

func disableRawMode() {
	if e := TcSetAttr(os.Stdin.Fd(), E.origTermios); e != nil {
		log.Fatalf("Problem disabling raw mode: %s\n", e)
	}
}

func editorReadKey() int {
	var buffer [1]byte
	var cc int
	var err error
	for cc, err = os.Stdin.Read(buffer[:]); cc != 1; cc, err = os.Stdin.Read(buffer[:]) {
	}
	if err != nil {
		die(err)
	}
	if buffer[0] == '\x1b' {
		var seq [2]byte
		if cc, _ = os.Stdin.Read(seq[:]); cc != 2 {
			return '\x1b'
		}

		if seq[0] == '[' {
			if seq[1] >= '0' && seq[1] <= '9' {
				if cc, err = os.Stdin.Read(buffer[:]); cc != 1 {
					return '\x1b'
				}
				if buffer[0] == '~' {
					switch seq[1] {
					case '1':
						return HOME_KEY
					case '3':
						return DEL_KEY
					case '4':
						return END_KEY
					case '5':
						return PAGE_UP
					case '6':
						return PAGE_DOWN
					case '7':
						return HOME_KEY
					case '8':
						return END_KEY
					}
				}
				// XXX - what happens here?
			} else {
				switch seq[1] {
				case 'A':
					return ARROW_UP
				case 'B':
					return ARROW_DOWN
				case 'C':
					return ARROW_RIGHT
				case 'D':
					return ARROW_LEFT
				case 'H':
					return HOME_KEY
				case 'F':
					return END_KEY
				}
			}
		} else if seq[0] == '0' {
			switch seq[1] {
			case 'H':
				return HOME_KEY
			case 'F':
				return END_KEY
			}
		}

		return '\x1b'
	}
	return int(buffer[0])
}

func getCursorPosition(rows *int, cols *int) int {
	io.WriteString(os.Stdout, "\x1b[6n")
	var buffer [1]byte
	var buf []byte
	var cc int
	for cc, _ = os.Stdin.Read(buffer[:]); cc == 1; cc, _ = os.Stdin.Read(buffer[:]) {
		if buffer[0] == 'R' {
			break
		}
		buf = append(buf, buffer[0])
	}
	if string(buf[0:2]) != "\x1b[" {
		log.Printf("Failed to read rows;cols from tty\n")
		return -1
	}
	if n, e := fmt.Sscanf(string(buf[2:]), "%d;%d", rows, cols); n != 2 || e != nil {
		if e != nil {
			log.Printf("getCursorPosition: fmt.Sscanf() failed: %s\n", e)
		}
		if n != 2 {
			log.Printf("getCursorPosition: got %d items, wanted 2\n", n)
		}
		return -1
	}
	return 0
}

func getWindowSize(rows *int, cols *int) int {
	var w WinSize
	_, _, err := syscall.Syscall(syscall.SYS_IOCTL,
		os.Stdout.Fd(),
		syscall.TIOCGWINSZ,
		uintptr(unsafe.Pointer(&w)),
	)
	if err != 0 { // type syscall.Errno
		io.WriteString(os.Stdout, "\x1b[999C\x1b[999B")
		return getCursorPosition(rows, cols)
	} else {
		*rows = int(w.Row)
		*cols = int(w.Col)
		return 0
	}
	return -1
}

/*** row operations ***/

func editorAppendRow(s []byte) {
	var r erow
	r.chars = s
	E.rows = append(E.rows, r)
	E.numRows++
}

/*** file I/O ***/

func editorOpen(filename string) {
	fd, err := os.Open(filename)
	if err != nil {
		die(err)
	}
	defer fd.Close()
	fp := bufio.NewReader(fd)

	for line, err := fp.ReadBytes('\n'); err == nil; line, err = fp.ReadBytes('\n') {
		// Trim trailing newlines and carriage returns
		for c := line[len(line) - 1]; len(line) > 0 && (c == '\n' || c == '\r'); {
			line = line[:len(line)-1]
			if len(line) > 0 {
				c = line[len(line) - 1]
			}
		}
		editorAppendRow(line)
	}

	if err != nil && err != io.EOF {
		die(err)
	}
}

/*** input ***/

func editorMoveCursor(key int) {
	switch key {
	case ARROW_LEFT:
		if E.cx != 0 {
			E.cx--
		}
	case ARROW_RIGHT:
		if E.cx != E.screenCols-1 {
			E.cx++
		}
	case ARROW_UP:
		if E.cy != 0 {
			E.cy--
		}
	case ARROW_DOWN:
		if E.cy < E.numRows {
			E.cy++
		}
	}
}

func editorProcessKeypress() {
	c := editorReadKey()
	switch c {
	case ('q' & 0x1f):
		io.WriteString(os.Stdout, "\x1b[2J")
		io.WriteString(os.Stdout, "\x1b[H")
		disableRawMode()
		os.Exit(0)
	case HOME_KEY:
		E.cx = 0
	case END_KEY:
		E.cx = E.screenCols - 1
	case PAGE_UP, PAGE_DOWN:
		dir := ARROW_DOWN
		if c == PAGE_UP {
			dir = ARROW_UP
		}
		for times := E.screenRows; times > 0; times-- {
			editorMoveCursor(dir)
		}
	case ARROW_UP, ARROW_DOWN, ARROW_LEFT, ARROW_RIGHT:
		editorMoveCursor(c)
	}
}

/*** append buffer ***/

type abuf struct {
	buf []byte
}

func (p abuf) String() string {
	return string(p.buf)
}

func (p *abuf) abAppend(s string) {
	p.buf = append(p.buf, []byte(s)...)
}

func (p *abuf) abAppendBytes(b []byte) {
	p.buf = append(p.buf, b...)
}

/*** output ***/

func editorScroll() {
	if E.cy < E.rowoff {
		E.rowoff = E.cy
	}
	if E.cy >= E.rowoff + E.screenRows {
		E.rowoff = E.cy - E.screenRows + 1
	}
}

func editorRefreshScreen() {
	editorScroll()
	var ab abuf
	ab.abAppend("\x1b[25l")
	ab.abAppend("\x1b[H")
	editorDrawRows(&ab)
	ab.abAppend(fmt.Sprintf("\x1b[%d;%dH", (E.cy - E.rowoff) + 1, E.cx+1))
	ab.abAppend("\x1b[25h")
	_, e := io.WriteString(os.Stdout, ab.String())
	if e != nil {
		log.Fatal(e)
	}
}

func editorDrawRows(ab *abuf) {
	for y := 0; y < E.screenRows; y++ {
		filerow := y + E.rowoff
		if filerow >= E.numRows {
			if E.numRows == 0 && y == E.screenRows/3 {
				w := fmt.Sprintf("Kilo editor -- version %s", KILO_VERSION)
				if len(w) > E.screenCols {
					w = w[0:E.screenCols]
				}
				pad := "~ "
				for padding := (E.screenCols - len(w)) / 2; padding > 0; padding-- {
					ab.abAppend(pad)
					pad = " "
				}
				ab.abAppend(w)
			} else {
				ab.abAppend("~")
			}
		} else {
			length := len(E.rows[filerow].chars)
			if length > E.screenCols { length = E.screenCols }
			ab.abAppendBytes(E.rows[filerow].chars[:length])
		}
		ab.abAppend("\x1b[K")
		if y < E.screenRows-1 {
			ab.abAppend("\r\n")
		}
	}
}

/*** init ***/

func initEditor() {
	// Initialization a la C not necessary.
	if getWindowSize(&E.screenRows, &E.screenCols) == -1 {
		die(fmt.Errorf("couldn't get screen size"))
	}
}

func main() {
	enableRawMode()
	defer disableRawMode()
	initEditor()
	if len(os.Args) > 1 {
		editorOpen(os.Args[1])
	}

	for {
		editorRefreshScreen()
		editorProcessKeypress()
	}
}
<commit_msg>Step 72<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"syscall"
	"unsafe"
)

/*** defines ***/

const KILO_VERSION = "0.0.1"
const (
	ARROW_LEFT  = 1000 + iota
	ARROW_RIGHT = 1000 + iota
	ARROW_UP    = 1000 + iota
	ARROW_DOWN  = 1000 + iota
	DEL_KEY     = 1000 + iota
	HOME_KEY    = 1000 + iota
	END_KEY     = 1000 + iota
	PAGE_UP     = 1000 + iota
	PAGE_DOWN   = 1000 + iota
)

/*** data ***/

type Termios struct {
	Iflag  uint32
	Oflag  uint32
	Cflag  uint32
	Lflag  uint32
	Cc     [20]byte
	Ispeed uint32
	Ospeed uint32
}

type erow struct {
	chars []byte
}

type editorConfig struct {
	cx          int
	cy          int
	rowoff      int
	coloff      int
	screenRows  int
	screenCols  int
	numRows     int
	rows        []erow
	origTermios *Termios
}

type WinSize struct {
	Row    uint16
	Col    uint16
	Xpixel uint16
	Ypixel uint16
}

var E editorConfig

/*** terminal ***/

func die(err error) {
	disableRawMode()
	io.WriteString(os.Stdout, "\x1b[2J")
	io.WriteString(os.Stdout, "\x1b[H")
	log.Fatal(err)
}

func TcSetAttr(fd uintptr, termios *Termios) error {
	// TCSETS+1 == TCSETSW, because TCSAFLUSH doesn't exist
	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, uintptr(syscall.TCSETS+1), uintptr(unsafe.Pointer(termios))); err != 0 {
		return err
	}
	return nil
}

func TcGetAttr(fd uintptr) *Termios {
	var termios = &Termios{}
	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, syscall.TCGETS, uintptr(unsafe.Pointer(termios))); err != 0 {
		log.Fatalf("Problem getting terminal attributes: %s\n", err)
	}
	return termios
}

func enableRawMode() {
	E.origTermios = TcGetAttr(os.Stdin.Fd())
	var raw Termios
	raw = *E.origTermios
	raw.Iflag &^= syscall.BRKINT | syscall.ICRNL | syscall.INPCK | syscall.ISTRIP | syscall.IXON
	raw.Oflag &^= syscall.OPOST
	raw.Cflag |= syscall.CS8
	raw.Lflag &^= syscall.ECHO | syscall.ICANON | syscall.IEXTEN | syscall.ISIG
	raw.Cc[syscall.VMIN+1] = 0
	raw.Cc[syscall.VTIME+1] = 1
	if e := TcSetAttr(os.Stdin.Fd(), &raw); e != nil {
		log.Fatalf("Problem enabling raw mode: %s\n", e)
	}
}

func disableRawMode() {
	if e := TcSetAttr(os.Stdin.Fd(), E.origTermios); e != nil {
		log.Fatalf("Problem disabling raw mode: %s\n", e)
	}
}

func editorReadKey() int {
	var buffer [1]byte
	var cc int
	var err error
	for cc, err = os.Stdin.Read(buffer[:]); cc != 1; cc, err = os.Stdin.Read(buffer[:]) {
	}
	if err != nil {
		die(err)
	}
	if buffer[0] == '\x1b' {
		var seq [2]byte
		if cc, _ = os.Stdin.Read(seq[:]); cc != 2 {
			return '\x1b'
		}

		if seq[0] == '[' {
			if seq[1] >= '0' && seq[1] <= '9' {
				if cc, err = os.Stdin.Read(buffer[:]); cc != 1 {
					return '\x1b'
				}
				if buffer[0] == '~' {
					switch seq[1] {
					case '1':
						return HOME_KEY
					case '3':
						return DEL_KEY
					case '4':
						return END_KEY
					case '5':
						return PAGE_UP
					case '6':
						return PAGE_DOWN
					case '7':
						return HOME_KEY
					case '8':
						return END_KEY
					}
				}
				// XXX - what happens here?
			} else {
				switch seq[1] {
				case 'A':
					return ARROW_UP
				case 'B':
					return ARROW_DOWN
				case 'C':
					return ARROW_RIGHT
				case 'D':
					return ARROW_LEFT
				case 'H':
					return HOME_KEY
				case 'F':
					return END_KEY
				}
			}
		} else if seq[0] == '0' {
			switch seq[1] {
			case 'H':
				return HOME_KEY
			case 'F':
				return END_KEY
			}
		}

		return '\x1b'
	}
	return int(buffer[0])
}

func getCursorPosition(rows *int, cols *int) int {
	io.WriteString(os.Stdout, "\x1b[6n")
	var buffer [1]byte
	var buf []byte
	var cc int
	for cc, _ = os.Stdin.Read(buffer[:]); cc == 1; cc, _ = os.Stdin.Read(buffer[:]) {
		if buffer[0] == 'R' {
			break
		}
		buf = append(buf, buffer[0])
	}
	if string(buf[0:2]) != "\x1b[" {
		log.Printf("Failed to read rows;cols from tty\n")
		return -1
	}
	if n, e := fmt.Sscanf(string(buf[2:]), "%d;%d", rows, cols); n != 2 || e != nil {
		if e != nil {
			log.Printf("getCursorPosition: fmt.Sscanf() failed: %s\n", e)
		}
		if n != 2 {
			log.Printf("getCursorPosition: got %d items, wanted 2\n", n)
		}
		return -1
	}
	return 0
}

func getWindowSize(rows *int, cols *int) int {
	var w WinSize
	_, _, err := syscall.Syscall(syscall.SYS_IOCTL,
		os.Stdout.Fd(),
		syscall.TIOCGWINSZ,
		uintptr(unsafe.Pointer(&w)),
	)
	if err != 0 { // type syscall.Errno
		io.WriteString(os.Stdout, "\x1b[999C\x1b[999B")
		return getCursorPosition(rows, cols)
	} else {
		*rows = int(w.Row)
		*cols = int(w.Col)
		return 0
	}
	return -1
}

/*** row operations ***/

func editorAppendRow(s []byte) {
	var r erow
	r.chars = s
	E.rows = append(E.rows, r)
	E.numRows++
}

/*** file I/O ***/

func editorOpen(filename string) {
	fd, err := os.Open(filename)
	if err != nil {
		die(err)
	}
	defer fd.Close()
	fp := bufio.NewReader(fd)

	for line, err := fp.ReadBytes('\n'); err == nil; line, err = fp.ReadBytes('\n') {
		// Trim trailing newlines and carriage returns
		for c := line[len(line) - 1]; len(line) > 0 && (c == '\n' || c == '\r'); {
			line = line[:len(line)-1]
			if len(line) > 0 {
				c = line[len(line) - 1]
			}
		}
		editorAppendRow(line)
	}

	if err != nil && err != io.EOF {
		die(err)
	}
}

/*** input ***/

func editorMoveCursor(key int) {
	switch key {
	case ARROW_LEFT:
		if E.cx != 0 {
			E.cx--
		}
	case ARROW_RIGHT:
		if E.cx != E.screenCols-1 {
			E.cx++
		}
	case ARROW_UP:
		if E.cy != 0 {
			E.cy--
		}
	case ARROW_DOWN:
		if E.cy < E.numRows {
			E.cy++
		}
	}
}

func editorProcessKeypress() {
	c := editorReadKey()
	switch c {
	case ('q' & 0x1f):
		io.WriteString(os.Stdout, "\x1b[2J")
		io.WriteString(os.Stdout, "\x1b[H")
		disableRawMode()
		os.Exit(0)
	case HOME_KEY:
		E.cx = 0
	case END_KEY:
		E.cx = E.screenCols - 1
	case PAGE_UP, PAGE_DOWN:
		dir := ARROW_DOWN
		if c == PAGE_UP {
			dir = ARROW_UP
		}
		for times := E.screenRows; times > 0; times-- {
			editorMoveCursor(dir)
		}
	case ARROW_UP, ARROW_DOWN, ARROW_LEFT, ARROW_RIGHT:
		editorMoveCursor(c)
	}
}

/*** append buffer ***/

type abuf struct {
	buf []byte
}

func (p abuf) String() string {
	return string(p.buf)
}

func (p *abuf) abAppend(s string) {
	p.buf = append(p.buf, []byte(s)...)
}

func (p *abuf) abAppendBytes(b []byte) {
	p.buf = append(p.buf, b...)
}

/*** output ***/

func editorScroll() {
	if E.cy < E.rowoff {
		E.rowoff = E.cy
	}
	if E.cy >= E.rowoff + E.screenRows {
		E.rowoff = E.cy - E.screenRows + 1
	}
}

func editorRefreshScreen() {
	editorScroll()
	var ab abuf
	ab.abAppend("\x1b[25l")
	ab.abAppend("\x1b[H")
	editorDrawRows(&ab)
	ab.abAppend(fmt.Sprintf("\x1b[%d;%dH", (E.cy - E.rowoff) + 1, E.cx+1))
	ab.abAppend("\x1b[25h")
	_, e := io.WriteString(os.Stdout, ab.String())
	if e != nil {
		log.Fatal(e)
	}
}

func editorDrawRows(ab *abuf) {
	for y := 0; y < E.screenRows; y++ {
		filerow := y + E.rowoff
		if filerow >= E.numRows {
			if E.numRows == 0 && y == E.screenRows/3 {
				w := fmt.Sprintf("Kilo editor -- version %s", KILO_VERSION)
				if len(w) > E.screenCols {
					w = w[0:E.screenCols]
				}
				pad := "~ "
				for padding := (E.screenCols - len(w)) / 2; padding > 0; padding-- {
					ab.abAppend(pad)
					pad = " "
				}
				ab.abAppend(w)
			} else {
				ab.abAppend("~")
			}
		} else {
			length := len(E.rows[filerow].chars) - E.coloff
			if length < 0 { length = 0 }
			if length > E.screenCols { length = E.screenCols }
			ab.abAppendBytes(E.rows[filerow].chars[E.coloff:length])
		}
		ab.abAppend("\x1b[K")
		if y < E.screenRows-1 {
			ab.abAppend("\r\n")
		}
	}
}

/*** init ***/

func initEditor() {
	// Initialization a la C not necessary.
	if getWindowSize(&E.screenRows, &E.screenCols) == -1 {
		die(fmt.Errorf("couldn't get screen size"))
	}
}

func main() {
	enableRawMode()
	defer disableRawMode()
	initEditor()
	if len(os.Args) > 1 {
		editorOpen(os.Args[1])
	}

	for {
		editorRefreshScreen()
		editorProcessKeypress()
	}
}
<|endoftext|>