<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package etcd3

import (
	"context"
	"errors"
	"fmt"
	"os"
	"strconv"
	"strings"
	"sync"

	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/apiserver/pkg/storage"
	"k8s.io/apiserver/pkg/storage/value"

	"go.etcd.io/etcd/clientv3"
	"k8s.io/klog"
)

const (
	// We have set a buffer in order to reduce times of context switches.
	incomingBufSize = 100
	outgoingBufSize = 100
)

// fatalOnDecodeError is used during testing to panic the server if watcher encounters a decoding error
var fatalOnDecodeError = false

// errTestingDecode is the only error that testingDeferOnDecodeError catches during a panic
var errTestingDecode = errors.New("sentinel error only used during testing to indicate watch decoding error")

// testingDeferOnDecodeError is used during testing to recover from a panic caused by errTestingDecode, all other values continue to panic
func testingDeferOnDecodeError() {
	if r := recover(); r != nil && r != errTestingDecode {
		panic(r)
	}
}

func init() {
	// check to see if we are running in a test environment
	TestOnlySetFatalOnDecodeError(true)
	fatalOnDecodeError, _ = strconv.ParseBool(os.Getenv("KUBE_PANIC_WATCH_DECODE_ERROR"))
}

// TestOnlySetFatalOnDecodeError should only be used for cases where decode errors are expected and need to be tested. e.g. conversion webhooks.
func TestOnlySetFatalOnDecodeError(b bool) {
	fatalOnDecodeError = b
}

type watcher struct {
	client      *clientv3.Client
	codec       runtime.Codec
	versioner   storage.Versioner
	transformer value.Transformer
}

// watchChan implements watch.Interface.
type watchChan struct {
	watcher           *watcher
	key               string
	initialRev        int64
	recursive         bool
	internalPred      storage.SelectionPredicate
	ctx               context.Context
	cancel            context.CancelFunc
	incomingEventChan chan *event
	resultChan        chan watch.Event
	errChan           chan error
}

func newWatcher(client *clientv3.Client, codec runtime.Codec, versioner storage.Versioner, transformer value.Transformer) *watcher {
	return &watcher{
		client:      client,
		codec:       codec,
		versioner:   versioner,
		transformer: transformer,
	}
}

// Watch watches on a key and returns a watch.Interface that transfers relevant notifications.
// If rev is zero, it will return the existing object(s) and then start watching from
// the maximum revision+1 from returned objects.
// If rev is non-zero, it will watch events happened after given revision.
// If recursive is false, it watches on given key.
// If recursive is true, it watches any children and directories under the key, excluding the root key itself.
// pred must be non-nil. Only if pred matches the change, it will be returned.
func (w *watcher) Watch(ctx context.Context, key string, rev int64, recursive bool, pred storage.SelectionPredicate) (watch.Interface, error) {
	if recursive && !strings.HasSuffix(key, "/") {
		key += "/"
	}
	wc := w.createWatchChan(ctx, key, rev, recursive, pred)
	go wc.run()
	return wc, nil
}

func (w *watcher) createWatchChan(ctx context.Context, key string, rev int64, recursive bool, pred storage.SelectionPredicate) *watchChan {
	wc := &watchChan{
		watcher:           w,
		key:               key,
		initialRev:        rev,
		recursive:         recursive,
		internalPred:      pred,
		incomingEventChan: make(chan *event, incomingBufSize),
		resultChan:        make(chan watch.Event, outgoingBufSize),
		errChan:           make(chan error, 1),
	}
	if pred.Empty() {
		// The filter doesn't filter out any object.
		wc.internalPred = storage.Everything
	}
	wc.ctx, wc.cancel = context.WithCancel(ctx)
	return wc
}

func (wc *watchChan) run() {
	watchClosedCh := make(chan struct{})
	go wc.startWatching(watchClosedCh)

	var resultChanWG sync.WaitGroup
	resultChanWG.Add(1)
	go wc.processEvent(&resultChanWG)

	select {
	case err := <-wc.errChan:
		if err == context.Canceled {
			break
		}
		errResult := transformErrorToEvent(err)
		if errResult != nil {
			// error result is guaranteed to be received by user before closing ResultChan.
			select {
			case wc.resultChan <- *errResult:
			case <-wc.ctx.Done(): // user has given up all results
			}
		}
	case <-watchClosedCh:
	case <-wc.ctx.Done(): // user cancel
	}

	// We use wc.ctx to reap all goroutines. Under whatever condition, we should stop them all.
	// It's fine to double cancel.
	wc.cancel()

	// we need to wait until resultChan wouldn't be used anymore
	resultChanWG.Wait()
	close(wc.resultChan)
}

func (wc *watchChan) Stop() {
	wc.cancel()
}

func (wc *watchChan) ResultChan() <-chan watch.Event {
	return wc.resultChan
}

// sync tries to retrieve existing data and send them to process.
// The revision to watch will be set to the revision in response.
// All events sent will have isCreated=true
func (wc *watchChan) sync() error {
	opts := []clientv3.OpOption{}
	if wc.recursive {
		opts = append(opts, clientv3.WithPrefix())
	}
	getResp, err := wc.watcher.client.Get(wc.ctx, wc.key, opts...)
	if err != nil {
		return err
	}
	wc.initialRev = getResp.Header.Revision
	for _, kv := range getResp.Kvs {
		wc.sendEvent(parseKV(kv))
	}
	return nil
}

// logWatchChannelErr checks whether the error is about mvcc revision compaction which is regarded as warning
func logWatchChannelErr(err error) {
	if !strings.Contains(err.Error(), "mvcc: required revision has been compacted") {
		klog.Errorf("watch chan error: %v", err)
	} else {
		klog.Warningf("watch chan error: %v", err)
	}
}

// startWatching does:
// - get current objects if initialRev=0; set initialRev to current rev
// - watch on given key and send events to process.
func (wc *watchChan) startWatching(watchClosedCh chan struct{}) {
	if wc.initialRev == 0 {
		if err := wc.sync(); err != nil {
			klog.Errorf("failed to sync with latest state: %v", err)
			wc.sendError(err)
			return
		}
	}
	opts := []clientv3.OpOption{clientv3.WithRev(wc.initialRev + 1), clientv3.WithPrevKV()}
	if wc.recursive {
		opts = append(opts, clientv3.WithPrefix())
	}
	wch := wc.watcher.client.Watch(wc.ctx, wc.key, opts...)
	for wres := range wch {
		if wres.Err() != nil {
			err := wres.Err()
			// If there is an error on server (e.g. compaction), the channel will return it before closed.
			logWatchChannelErr(err)
			wc.sendError(err)
			return
		}
		for _, e := range wres.Events {
			parsedEvent, err := parseEvent(e)
			if err != nil {
				logWatchChannelErr(err)
				wc.sendError(err)
				return
			}
			wc.sendEvent(parsedEvent)
		}
	}
	// When we come to this point, it's only possible that client side ends the watch.
	// e.g. cancel the context, close the client.
	// If this watch chan is broken and context isn't cancelled, other goroutines will still hang.
	// We should notify the main thread that this goroutine has exited.
	close(watchClosedCh)
}

// processEvent processes events from etcd watcher and sends results to resultChan.
func (wc *watchChan) processEvent(wg *sync.WaitGroup) {
	defer wg.Done()

	for {
		select {
		case e := <-wc.incomingEventChan:
			res := wc.transform(e)
			if res == nil {
				continue
			}
			if len(wc.resultChan) == outgoingBufSize {
				klog.V(3).Infof("Fast watcher, slow processing. Number of buffered events: %d."+
					"Probably caused by slow dispatching events to watchers", outgoingBufSize)
			}
			// If user couldn't receive results fast enough, we also block incoming events from watcher.
			// Because storing events in local will cause more memory usage.
			// The worst case would be closing the fast watcher.
			select {
			case wc.resultChan <- *res:
			case <-wc.ctx.Done():
				return
			}
		case <-wc.ctx.Done():
			return
		}
	}
}

func (wc *watchChan) filter(obj runtime.Object) bool {
	if wc.internalPred.Empty() {
		return true
	}
	matched, err := wc.internalPred.Matches(obj)
	return err == nil && matched
}

func (wc *watchChan) acceptAll() bool {
	return wc.internalPred.Empty()
}

// transform transforms an event into a result for user if not filtered.
func (wc *watchChan) transform(e *event) (res *watch.Event) {
	curObj, oldObj, err := wc.prepareObjs(e)
	if err != nil {
		klog.Errorf("failed to prepare current and previous objects: %v", err)
		wc.sendError(err)
		return nil
	}

	switch {
	case e.isDeleted:
		if !wc.filter(oldObj) {
			return nil
		}
		res = &watch.Event{
			Type:   watch.Deleted,
			Object: oldObj,
		}
	case e.isCreated:
		if !wc.filter(curObj) {
			return nil
		}
		res = &watch.Event{
			Type:   watch.Added,
			Object: curObj,
		}
	default:
		if wc.acceptAll() {
			res = &watch.Event{
				Type:   watch.Modified,
				Object: curObj,
			}
			return res
		}
		curObjPasses := wc.filter(curObj)
		oldObjPasses := wc.filter(oldObj)
		switch {
		case curObjPasses && oldObjPasses:
			res = &watch.Event{
				Type:   watch.Modified,
				Object: curObj,
			}
		case curObjPasses && !oldObjPasses:
			res = &watch.Event{
				Type:   watch.Added,
				Object: curObj,
			}
		case !curObjPasses && oldObjPasses:
			res = &watch.Event{
				Type:   watch.Deleted,
				Object: oldObj,
			}
		}
	}
	return res
}

func transformErrorToEvent(err error) *watch.Event {
	err = interpretWatchError(err)
	if _, ok := err.(apierrors.APIStatus); !ok {
		err = apierrors.NewInternalError(err)
	}
	status := err.(apierrors.APIStatus).Status()
	return &watch.Event{
		Type:   watch.Error,
		Object: &status,
	}
}

func (wc *watchChan) sendError(err error) {
	select {
	case wc.errChan <- err:
	case <-wc.ctx.Done():
	}
}

func (wc *watchChan) sendEvent(e *event) {
	if len(wc.incomingEventChan) == incomingBufSize {
		klog.V(3).Infof("Fast watcher, slow processing. Number of buffered events: %d."+
			"Probably caused by slow decoding, user not receiving fast, or other processing logic",
			incomingBufSize)
	}
	select {
	case wc.incomingEventChan <- e:
	case <-wc.ctx.Done():
	}
}

func (wc *watchChan) prepareObjs(e *event) (curObj runtime.Object, oldObj runtime.Object, err error) {
	if !e.isDeleted {
		data, _, err := wc.watcher.transformer.TransformFromStorage(e.value, authenticatedDataString(e.key))
		if err != nil {
			return nil, nil, err
		}
		curObj, err = decodeObj(wc.watcher.codec, wc.watcher.versioner, data, e.rev)
		if err != nil {
			return nil, nil, err
		}
	}
	// We need to decode prevValue, only if this is deletion event or
	// the underlying filter doesn't accept all objects (otherwise we
	// know that the filter for previous object will return true and
	// we need the object only to compute whether it was filtered out
	// before).
	if len(e.prevValue) > 0 && (e.isDeleted || !wc.acceptAll()) {
		data, _, err := wc.watcher.transformer.TransformFromStorage(e.prevValue, authenticatedDataString(e.key))
		if err != nil {
			return nil, nil, err
		}
		// Note that this sends the *old* object with the etcd revision for the time at
		// which it gets deleted.
		oldObj, err = decodeObj(wc.watcher.codec, wc.watcher.versioner, data, e.rev)
		if err != nil {
			return nil, nil, err
		}
	}
	return curObj, oldObj, nil
}

func decodeObj(codec runtime.Codec, versioner storage.Versioner, data []byte, rev int64) (_ runtime.Object, err error) {
	obj, err := runtime.Decode(codec, []byte(data))
	if err != nil {
		if fatalOnDecodeError {
			// catch watch decode error iff we caused it on
			// purpose during a unit test
			defer testingDeferOnDecodeError()
			// we are running in a test environment and thus an
			// error here is due to a coder mistake if the defer
			// does not catch it
			panic(err)
		}
		return nil, err
	}
	// ensure resource version is set on the object we load from etcd
	if err := versioner.UpdateObject(obj, uint64(rev)); err != nil {
		return nil, fmt.Errorf("failure to version api object (%d) %#v: %v", rev, obj, err)
	}
	return obj, nil
}
<commit_msg>Add etcd WithRequireLeader option to API watches<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package etcd3

import (
	"context"
	"errors"
	"fmt"
	"os"
	"strconv"
	"strings"
	"sync"

	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/apiserver/pkg/storage"
	"k8s.io/apiserver/pkg/storage/value"

	"go.etcd.io/etcd/clientv3"
	"k8s.io/klog"
)

const (
	// We have set a buffer in order to reduce times of context switches.
	incomingBufSize = 100
	outgoingBufSize = 100
)

// fatalOnDecodeError is used during testing to panic the server if watcher encounters a decoding error
var fatalOnDecodeError = false

// errTestingDecode is the only error that testingDeferOnDecodeError catches during a panic
var errTestingDecode = errors.New("sentinel error only used during testing to indicate watch decoding error")

// testingDeferOnDecodeError is used during testing to recover from a panic caused by errTestingDecode, all other values continue to panic
func testingDeferOnDecodeError() {
	if r := recover(); r != nil && r != errTestingDecode {
		panic(r)
	}
}

func init() {
	// check to see if we are running in a test environment
	TestOnlySetFatalOnDecodeError(true)
	fatalOnDecodeError, _ = strconv.ParseBool(os.Getenv("KUBE_PANIC_WATCH_DECODE_ERROR"))
}

// TestOnlySetFatalOnDecodeError should only be used for cases where decode errors are expected and need to be tested. e.g. conversion webhooks.
func TestOnlySetFatalOnDecodeError(b bool) {
	fatalOnDecodeError = b
}

type watcher struct {
	client      *clientv3.Client
	codec       runtime.Codec
	versioner   storage.Versioner
	transformer value.Transformer
}

// watchChan implements watch.Interface.
type watchChan struct {
	watcher           *watcher
	key               string
	initialRev        int64
	recursive         bool
	internalPred      storage.SelectionPredicate
	ctx               context.Context
	cancel            context.CancelFunc
	incomingEventChan chan *event
	resultChan        chan watch.Event
	errChan           chan error
}

func newWatcher(client *clientv3.Client, codec runtime.Codec, versioner storage.Versioner, transformer value.Transformer) *watcher {
	return &watcher{
		client:      client,
		codec:       codec,
		versioner:   versioner,
		transformer: transformer,
	}
}

// Watch watches on a key and returns a watch.Interface that transfers relevant notifications.
// If rev is zero, it will return the existing object(s) and then start watching from
// the maximum revision+1 from returned objects.
// If rev is non-zero, it will watch events happened after given revision.
// If recursive is false, it watches on given key.
// If recursive is true, it watches any children and directories under the key, excluding the root key itself.
// pred must be non-nil. Only if pred matches the change, it will be returned.
func (w *watcher) Watch(ctx context.Context, key string, rev int64, recursive bool, pred storage.SelectionPredicate) (watch.Interface, error) {
	if recursive && !strings.HasSuffix(key, "/") {
		key += "/"
	}
	wc := w.createWatchChan(ctx, key, rev, recursive, pred)
	go wc.run()
	return wc, nil
}

func (w *watcher) createWatchChan(ctx context.Context, key string, rev int64, recursive bool, pred storage.SelectionPredicate) *watchChan {
	wc := &watchChan{
		watcher:           w,
		key:               key,
		initialRev:        rev,
		recursive:         recursive,
		internalPred:      pred,
		incomingEventChan: make(chan *event, incomingBufSize),
		resultChan:        make(chan watch.Event, outgoingBufSize),
		errChan:           make(chan error, 1),
	}
	if pred.Empty() {
		// The filter doesn't filter out any object.
		wc.internalPred = storage.Everything
	}

	// The etcd server waits until it cannot find a leader for 3 election
	// timeouts to cancel existing streams. 3 is currently a hard coded
	// constant. The election timeout defaults to 1000ms. If the cluster is
	// healthy, when the leader is stopped, the leadership transfer should be
	// smooth. (leader transfers its leadership before stopping). If leader is
	// hard killed, other servers will take an election timeout to realize
	// leader lost and start campaign.
	wc.ctx, wc.cancel = context.WithCancel(clientv3.WithRequireLeader(ctx))
	return wc
}

func (wc *watchChan) run() {
	watchClosedCh := make(chan struct{})
	go wc.startWatching(watchClosedCh)

	var resultChanWG sync.WaitGroup
	resultChanWG.Add(1)
	go wc.processEvent(&resultChanWG)

	select {
	case err := <-wc.errChan:
		if err == context.Canceled {
			break
		}
		errResult := transformErrorToEvent(err)
		if errResult != nil {
			// error result is guaranteed to be received by user before closing ResultChan.
			select {
			case wc.resultChan <- *errResult:
			case <-wc.ctx.Done(): // user has given up all results
			}
		}
	case <-watchClosedCh:
	case <-wc.ctx.Done(): // user cancel
	}

	// We use wc.ctx to reap all goroutines. Under whatever condition, we should stop them all.
	// It's fine to double cancel.
	wc.cancel()

	// we need to wait until resultChan wouldn't be used anymore
	resultChanWG.Wait()
	close(wc.resultChan)
}

func (wc *watchChan) Stop() {
	wc.cancel()
}

func (wc *watchChan) ResultChan() <-chan watch.Event {
	return wc.resultChan
}

// sync tries to retrieve existing data and send them to process.
// The revision to watch will be set to the revision in response.
// All events sent will have isCreated=true
func (wc *watchChan) sync() error {
	opts := []clientv3.OpOption{}
	if wc.recursive {
		opts = append(opts, clientv3.WithPrefix())
	}
	getResp, err := wc.watcher.client.Get(wc.ctx, wc.key, opts...)
	if err != nil {
		return err
	}
	wc.initialRev = getResp.Header.Revision
	for _, kv := range getResp.Kvs {
		wc.sendEvent(parseKV(kv))
	}
	return nil
}

// logWatchChannelErr checks whether the error is about mvcc revision compaction which is regarded as warning
func logWatchChannelErr(err error) {
	if !strings.Contains(err.Error(), "mvcc: required revision has been compacted") {
		klog.Errorf("watch chan error: %v", err)
	} else {
		klog.Warningf("watch chan error: %v", err)
	}
}

// startWatching does:
// - get current objects if initialRev=0; set initialRev to current rev
// - watch on given key and send events to process.
func (wc *watchChan) startWatching(watchClosedCh chan struct{}) {
	if wc.initialRev == 0 {
		if err := wc.sync(); err != nil {
			klog.Errorf("failed to sync with latest state: %v", err)
			wc.sendError(err)
			return
		}
	}
	opts := []clientv3.OpOption{clientv3.WithRev(wc.initialRev + 1), clientv3.WithPrevKV()}
	if wc.recursive {
		opts = append(opts, clientv3.WithPrefix())
	}
	wch := wc.watcher.client.Watch(wc.ctx, wc.key, opts...)
	for wres := range wch {
		if wres.Err() != nil {
			err := wres.Err()
			// If there is an error on server (e.g. compaction), the channel will return it before closed.
			logWatchChannelErr(err)
			wc.sendError(err)
			return
		}
		for _, e := range wres.Events {
			parsedEvent, err := parseEvent(e)
			if err != nil {
				logWatchChannelErr(err)
				wc.sendError(err)
				return
			}
			wc.sendEvent(parsedEvent)
		}
	}
	// When we come to this point, it's only possible that client side ends the watch.
	// e.g. cancel the context, close the client.
	// If this watch chan is broken and context isn't cancelled, other goroutines will still hang.
	// We should notify the main thread that this goroutine has exited.
	close(watchClosedCh)
}

// processEvent processes events from etcd watcher and sends results to resultChan.
func (wc *watchChan) processEvent(wg *sync.WaitGroup) {
	defer wg.Done()

	for {
		select {
		case e := <-wc.incomingEventChan:
			res := wc.transform(e)
			if res == nil {
				continue
			}
			if len(wc.resultChan) == outgoingBufSize {
				klog.V(3).Infof("Fast watcher, slow processing. Number of buffered events: %d."+
					"Probably caused by slow dispatching events to watchers", outgoingBufSize)
			}
			// If user couldn't receive results fast enough, we also block incoming events from watcher.
			// Because storing events in local will cause more memory usage.
			// The worst case would be closing the fast watcher.
			select {
			case wc.resultChan <- *res:
			case <-wc.ctx.Done():
				return
			}
		case <-wc.ctx.Done():
			return
		}
	}
}

func (wc *watchChan) filter(obj runtime.Object) bool {
	if wc.internalPred.Empty() {
		return true
	}
	matched, err := wc.internalPred.Matches(obj)
	return err == nil && matched
}

func (wc *watchChan) acceptAll() bool {
	return wc.internalPred.Empty()
}

// transform transforms an event into a result for user if not filtered.
func (wc *watchChan) transform(e *event) (res *watch.Event) {
	curObj, oldObj, err := wc.prepareObjs(e)
	if err != nil {
		klog.Errorf("failed to prepare current and previous objects: %v", err)
		wc.sendError(err)
		return nil
	}

	switch {
	case e.isDeleted:
		if !wc.filter(oldObj) {
			return nil
		}
		res = &watch.Event{
			Type:   watch.Deleted,
			Object: oldObj,
		}
	case e.isCreated:
		if !wc.filter(curObj) {
			return nil
		}
		res = &watch.Event{
			Type:   watch.Added,
			Object: curObj,
		}
	default:
		if wc.acceptAll() {
			res = &watch.Event{
				Type:   watch.Modified,
				Object: curObj,
			}
			return res
		}
		curObjPasses := wc.filter(curObj)
		oldObjPasses := wc.filter(oldObj)
		switch {
		case curObjPasses && oldObjPasses:
			res = &watch.Event{
				Type:   watch.Modified,
				Object: curObj,
			}
		case curObjPasses && !oldObjPasses:
			res = &watch.Event{
				Type:   watch.Added,
				Object: curObj,
			}
		case !curObjPasses && oldObjPasses:
			res = &watch.Event{
				Type:   watch.Deleted,
				Object: oldObj,
			}
		}
	}
	return res
}

func transformErrorToEvent(err error) *watch.Event {
	err = interpretWatchError(err)
	if _, ok := err.(apierrors.APIStatus); !ok {
		err = apierrors.NewInternalError(err)
	}
	status := err.(apierrors.APIStatus).Status()
	return &watch.Event{
		Type:   watch.Error,
		Object: &status,
	}
}

func (wc *watchChan) sendError(err error) {
	select {
	case wc.errChan <- err:
	case <-wc.ctx.Done():
	}
}

func (wc *watchChan) sendEvent(e *event) {
	if len(wc.incomingEventChan) == incomingBufSize {
		klog.V(3).Infof("Fast watcher, slow processing. Number of buffered events: %d."+
			"Probably caused by slow decoding, user not receiving fast, or other processing logic",
			incomingBufSize)
	}
	select {
	case wc.incomingEventChan <- e:
	case <-wc.ctx.Done():
	}
}

func (wc *watchChan) prepareObjs(e *event) (curObj runtime.Object, oldObj runtime.Object, err error) {
	if !e.isDeleted {
		data, _, err := wc.watcher.transformer.TransformFromStorage(e.value, authenticatedDataString(e.key))
		if err != nil {
			return nil, nil, err
		}
		curObj, err = decodeObj(wc.watcher.codec, wc.watcher.versioner, data, e.rev)
		if err != nil {
			return nil, nil, err
		}
	}
	// We need to decode prevValue, only if this is deletion event or
	// the underlying filter doesn't accept all objects (otherwise we
	// know that the filter for previous object will return true and
	// we need the object only to compute whether it was filtered out
	// before).
	if len(e.prevValue) > 0 && (e.isDeleted || !wc.acceptAll()) {
		data, _, err := wc.watcher.transformer.TransformFromStorage(e.prevValue, authenticatedDataString(e.key))
		if err != nil {
			return nil, nil, err
		}
		// Note that this sends the *old* object with the etcd revision for the time at
		// which it gets deleted.
		oldObj, err = decodeObj(wc.watcher.codec, wc.watcher.versioner, data, e.rev)
		if err != nil {
			return nil, nil, err
		}
	}
	return curObj, oldObj, nil
}

func decodeObj(codec runtime.Codec, versioner storage.Versioner, data []byte, rev int64) (_ runtime.Object, err error) {
	obj, err := runtime.Decode(codec, []byte(data))
	if err != nil {
		if fatalOnDecodeError {
			// catch watch decode error iff we caused it on
			// purpose during a unit test
			defer testingDeferOnDecodeError()
			// we are running in a test environment and thus an
			// error here is due to a coder mistake if the defer
			// does not catch it
			panic(err)
		}
		return nil, err
	}
	// ensure resource version is set on the object we load from etcd
	if err := versioner.UpdateObject(obj, uint64(rev)); err != nil {
		return nil, fmt.Errorf("failure to version api object (%d) %#v: %v", rev, obj, err)
	}
	return obj, nil
}
<|endoftext|>


<commit_before>// This file is part of go-utils.
//
// Copyright (C) 2015-2016  David Gamba Rios
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

/*
Package fileutils - file related utilities
*/
package fileutils

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
)

// CopyFile copies the contents of the file named src to the file named
// by dst. The file will be created if it does not already exist. If the
// destination file exists, all it's contents will be replaced by the contents
// of the source file.
func CopyFile(src, dst string) error {
	fmt.Printf("Copy: %s %s\n", src, dst)
	in, err := os.Open(src)
	if err != nil {
		return err
	}
	defer in.Close()
	out, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer func() {
		cerr := out.Close()
		if err == nil {
			err = cerr
		}
	}()
	if _, err = io.Copy(out, in); err != nil {
		return err
	}
	err = out.Sync()
	return err
}

// ChannelString is a struct containing the string `Data` and error `Error`.
type ChannelString struct {
	Data  string
	Error error
}

// GetFileList returns a channel with each file (`channel.string`) or an error indicating failure (`channel.error`).
func GetFileList(filename string, ignoreDirs bool) <-chan ChannelString {
	c := make(chan ChannelString)
	go func() {
		fInfo, err := os.Stat(filename)
		if err != nil {
			c <- ChannelString{"", err}
			return
		}
		if fInfo.IsDir() {
			if ignoreDirs == false {
				c <- ChannelString{filename, nil}
			}
			fileSearch := filename + string(filepath.Separator) + "*"
			fileMatches, err := filepath.Glob(fileSearch)
			if err != nil {
				c <- ChannelString{"", err}
				return
			}
			for _, file := range fileMatches {
				if filepath.Base(filename) == filepath.Base(file) {
					continue
				}
				d := GetFileList(file, ignoreDirs)
				for dirFile := range d {
					if dirFile.Error != nil {
						return
					}
					c <- ChannelString{dirFile.Data, nil}
				}
			}
		} else {
			c <- ChannelString{filename, nil}
		}
		close(c)
	}()
	return c
}
<commit_msg>Update fileutils<commit_after>// This file is part of go-utils.
//
// Copyright (C) 2015-2016  David Gamba Rios
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

/*
Package fileutils - file related utilities
*/
package fileutils

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
)

// CopyFile copies the contents of the file named src to the file named
// by dst. The file will be created if it does not already exist. If the
// destination file exists, all it's contents will be replaced by the contents
// of the source file.
func CopyFile(src, dst string) error {
	fmt.Printf("Copy: %s %s\n", src, dst)
	in, err := os.Open(src)
	if err != nil {
		return err
	}
	defer in.Close()
	out, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer func() {
		cerr := out.Close()
		if err == nil {
			err = cerr
		}
	}()
	if _, err = io.Copy(out, in); err != nil {
		return err
	}
	err = out.Sync()
	return err
}

// StringError is a struct containing the string `String` and error `Error`.
type StringError struct {
	String string
	Error  error
}

// GetFileList returns a channel with each file (`channel.String`) or an error indicating failure (`channel.Error`).
func GetFileList(filename string, ignoreDirs bool) <-chan StringError {
	c := make(chan StringError)
	go func() {
		fInfo, err := os.Stat(filename)
		if err != nil {
			c <- StringError{"", err}
			return
		}
		if fInfo.IsDir() {
			if ignoreDirs == false {
				c <- StringError{filename, nil}
			}
			fileSearch := filename + string(filepath.Separator) + "*"
			fileMatches, err := filepath.Glob(fileSearch)
			if err != nil {
				c <- StringError{"", err}
				return
			}
			for _, file := range fileMatches {
				if filepath.Base(filename) == filepath.Base(file) {
					continue
				}
				d := GetFileList(file, ignoreDirs)
				for dirFile := range d {
					if dirFile.Error != nil {
						return
					}
					c <- StringError{dirFile.String, nil}
				}
			}
		} else {
			c <- StringError{filename, nil}
		}
		close(c)
	}()
	return c
}

// GetDirList returns a channel with each file (`channel.String`) or an error indicating failure (`channel.Error`).
func GetDirList(dir string) <-chan StringError {
	c := make(chan StringError)
	go func() {
		fInfo, err := os.Stat(dir)
		if err != nil {
			c <- StringError{"", err}
			return
		}
		if fInfo.IsDir() {
			c <- StringError{dir, nil}
			fileSearch := dir + string(filepath.Separator) + "*"
			fileMatches, err := filepath.Glob(fileSearch)
			if err != nil {
				c <- StringError{"", err}
				return
			}
			for _, file := range fileMatches {
				if filepath.Base(dir) == filepath.Base(file) {
					continue
				}
				d := GetDirList(file)
				for dirFile := range d {
					if dirFile.Error != nil {
						return
					}
					c <- StringError{dirFile.String, nil}
				}
			}
		}
		close(c)
	}()
	return c
}
<|endoftext|>


<commit_before>package filesystem

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/user"

	"github.com/clebs/shelf/errorout"
	"github.com/clebs/shelf/project"
)

const shelfDir = ".shelf"

//Save persists the provided object in JSON format into the filesystem under "{user}/.shelf/{pID}.json"
func Save(p project.Project, pID string) {
	data, err := json.MarshalIndent(p, "", "    ")
	errorout.ErrQuit(err, "Could not serialize project.")

	err = ioutil.WriteFile(getProjectFilePath(pID), data, 0644)
	errorout.ErrQuit(err, "Could not add Project to the shelf")
}

//Load locates a project on the filesystem by its name and loads its data
func Load(name string) project.Project {
	filePath := getProjectFilePath(name)

	if _, err := os.Stat(filePath); os.IsNotExist(err) {
		errorout.ErrQuit(err, fmt.Sprintf("The project %s could not be found.", name))
	}

	projectData, err := ioutil.ReadFile(filePath)
	errorout.ErrQuit(err, fmt.Sprintf("Could not load project %s", name))

	var p project.Project
	err = json.Unmarshal(projectData, &p)
	errorout.ErrQuit(err, "Error parsing project data.")
	return p
}

//Delete removes a project setup file form the system
func Delete(name string) {
	file := getProjectFilePath(name)
	err := os.Remove(file)
	errorout.ErrQuit(err, "Could not delete project")
	fmt.Printf("The project %s has been removed from the shelf.", name)
}

func getProjectFilePath(name string) string {
	user, err := user.Current()
	errorout.ErrQuit(err, "Could not obtain user home directory.")

	projectsFolder := user.HomeDir + "/" + shelfDir
	if _, err := os.Stat(projectsFolder); os.IsNotExist(err) {
		err := os.Mkdir(projectsFolder, 0700)
		errorout.ErrQuit(err, "Could not create Shelf directory")
	}
	return projectsFolder + "/" + name + ".json"
}
<commit_msg>Added new line after removal confirmation<commit_after>package filesystem

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/user"

	"github.com/clebs/shelf/errorout"
	"github.com/clebs/shelf/project"
)

const shelfDir = ".shelf"

//Save persists the provided object in JSON format into the filesystem under "{user}/.shelf/{pID}.json"
func Save(p project.Project, pID string) {
	data, err := json.MarshalIndent(p, "", "    ")
	errorout.ErrQuit(err, "Could not serialize project.")

	err = ioutil.WriteFile(getProjectFilePath(pID), data, 0644)
	errorout.ErrQuit(err, "Could not add Project to the shelf")
}

//Load locates a project on the filesystem by its name and loads its data
func Load(name string) project.Project {
	filePath := getProjectFilePath(name)

	if _, err := os.Stat(filePath); os.IsNotExist(err) {
		errorout.ErrQuit(err, fmt.Sprintf("The project %s could not be found.", name))
	}

	projectData, err := ioutil.ReadFile(filePath)
	errorout.ErrQuit(err, fmt.Sprintf("Could not load project %s", name))

	var p project.Project
	err = json.Unmarshal(projectData, &p)
	errorout.ErrQuit(err, "Error parsing project data.")
	return p
}

//Delete removes a project setup file form the system
func Delete(name string) {
	file := getProjectFilePath(name)
	err := os.Remove(file)
	errorout.ErrQuit(err, "Could not delete project")
	fmt.Printf("The project %s has been removed from the shelf.\n", name)
}

func getProjectFilePath(name string) string {
	user, err := user.Current()
	errorout.ErrQuit(err, "Could not obtain user home directory.")

	projectsFolder := user.HomeDir + "/" + shelfDir
	if _, err := os.Stat(projectsFolder); os.IsNotExist(err) {
		err := os.Mkdir(projectsFolder, 0700)
		errorout.ErrQuit(err, "Could not create Shelf directory")
	}
	return projectsFolder + "/" + name + ".json"
}
<|endoftext|>


<commit_before>package model

import (
	"time"
)

type tasks struct {
	Id        string
	Name      string
	Point     float64
	StartDate time.Time
}
<commit_msg>add end time.<commit_after>package model

import (
	"time"
)

type tasks struct {
	Id        string
	Name      string
	Point     float64
	StartDate time.Time
	EndDate   time.Time
}
<|endoftext|>


<commit_before>package download_step

import (
	"archive/tar"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"time"

	"github.com/cloudfoundry-incubator/executor/steps/emittable_error"
	"github.com/cloudfoundry-incubator/garden/warden"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	steno "github.com/cloudfoundry/gosteno"
	"github.com/pivotal-golang/archiver/compressor"
	"github.com/pivotal-golang/archiver/extractor"
	"github.com/pivotal-golang/cacheddownloader"
)

type DownloadStep struct {
	container        warden.Container
	model            models.DownloadAction
	cachedDownloader cacheddownloader.CachedDownloader
	extractor        extractor.Extractor
	tempDir          string
	logger           *steno.Logger
}

func New(
	container warden.Container,
	model models.DownloadAction,
	cachedDownloader cacheddownloader.CachedDownloader,
	extractor extractor.Extractor,
	tempDir string,
	logger *steno.Logger,
) *DownloadStep {
	return &DownloadStep{
		container:        container,
		model:            model,
		cachedDownloader: cachedDownloader,
		extractor:        extractor,
		tempDir:          tempDir,
		logger:           logger,
	}
}

func (step *DownloadStep) Perform() error {
	step.logger.Infod(
		map[string]interface{}{
			"handle": step.container.Handle(),
		},
		"task.handle.download-action",
	)

	//Stream this to the extractor + container when we have streaming support!
	downloadedPath, err := step.download()
	if err != nil {
		step.logger.Errord(
			map[string]interface{}{
				"handle": step.container.Handle(),
				"from":   step.model.From,
				"to":     step.model.To,
				"error":  err,
			},
			"task.handle.download-failed",
		)

		return err
	}

	defer os.Remove(downloadedPath)

	if step.model.Extract {
		extractionDir, err := step.extract(downloadedPath)
		if err != nil {
			return emittable_error.New(err, "Extraction failed")
		}

		defer os.RemoveAll(extractionDir)

		err = step.copyExtractedFiles(extractionDir, step.model.To)
		if err != nil {
			return emittable_error.New(err, "Copying into the container failed")
		}

		return err
	} else {
		reader, writer := io.Pipe()
		defer reader.Close()

		downloadedFile, err := os.Open(downloadedPath)
		if err != nil {
			return err
		}

		defer downloadedFile.Close()

		go writeTarTo(filepath.Base(step.model.To), downloadedFile, writer)

		err = step.container.StreamIn(filepath.Dir(step.model.To), reader)
		if err != nil {
			return emittable_error.New(err, "Copying into the container failed")
		}

		return nil
	}
}

func (step *DownloadStep) download() (string, error) {
	url, err := url.ParseRequestURI(step.model.From)
	if err != nil {
		return "", err
	}

	tempFile, err := ioutil.TempFile(step.tempDir, "downloaded")
	if err != nil {
		return "", err
	}
	defer tempFile.Close()

	downloadedFile, err := step.cachedDownloader.Fetch(url, step.model.CacheKey)
	if err != nil {
		return "", err
	}
	defer downloadedFile.Close()

	_, err = io.Copy(tempFile, downloadedFile)
	if err != nil {
		return "", err
	}

	return tempFile.Name(), nil
}

func (step *DownloadStep) extract(downloadedPath string) (string, error) {
	extractionDir, err := ioutil.TempDir(step.tempDir, "extracted")
	if err != nil {
		return "", err
	}

	err = step.extractor.Extract(downloadedPath, extractionDir)
	if err != nil {
		step.logger.Warnd(
			map[string]interface{}{
				"error": err.Error(),
				"url":   step.model.From,
			},
			"downloader.extract-failed",
		)

		return "", err
	}

	return extractionDir, nil
}

func (step *DownloadStep) Cancel() {}

func (step *DownloadStep) Cleanup() {}

func (step *DownloadStep) copyExtractedFiles(source string, destination string) error {
	reader, writer := io.Pipe()

	go func() {
		compressor.WriteTar(source+string(filepath.Separator), writer)
		writer.Close()
	}()
	defer reader.Close()

	return step.container.StreamIn(destination, reader)
}

func writeTarTo(name string, source *os.File, destination io.WriteCloser) error {
	tarWriter := tar.NewWriter(destination)

	fileInfo, err := source.Stat()
	if err != nil {
		return err
	}

	err = tarWriter.WriteHeader(&tar.Header{
		Name:       name,
		Size:       fileInfo.Size(),
		Mode:       0644,
		AccessTime: time.Now(),
		ChangeTime: time.Now(),
	})
	if err != nil {
		return err
	}

	_, err = io.Copy(tarWriter, source)
	if err != nil {
		return err
	}

	if err := tarWriter.Flush(); err != nil {
		return err
	}

	if err := destination.Close(); err != nil {
		return err
	}

	return nil
}
<commit_msg>Open downloadedFile in goroutine to avoid race<commit_after>package download_step

import (
	"archive/tar"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"time"

	"github.com/cloudfoundry-incubator/executor/steps/emittable_error"
	"github.com/cloudfoundry-incubator/garden/warden"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	steno "github.com/cloudfoundry/gosteno"
	"github.com/pivotal-golang/archiver/compressor"
	"github.com/pivotal-golang/archiver/extractor"
	"github.com/pivotal-golang/cacheddownloader"
)

type DownloadStep struct {
	container        warden.Container
	model            models.DownloadAction
	cachedDownloader cacheddownloader.CachedDownloader
	extractor        extractor.Extractor
	tempDir          string
	logger           *steno.Logger
}

func New(
	container warden.Container,
	model models.DownloadAction,
	cachedDownloader cacheddownloader.CachedDownloader,
	extractor extractor.Extractor,
	tempDir string,
	logger *steno.Logger,
) *DownloadStep {
	return &DownloadStep{
		container:        container,
		model:            model,
		cachedDownloader: cachedDownloader,
		extractor:        extractor,
		tempDir:          tempDir,
		logger:           logger,
	}
}

func (step *DownloadStep) Perform() error {
	step.logger.Infod(
		map[string]interface{}{
			"handle": step.container.Handle(),
		},
		"task.handle.download-action",
	)

	//Stream this to the extractor + container when we have streaming support!
	downloadedPath, err := step.download()
	if err != nil {
		step.logger.Errord(
			map[string]interface{}{
				"handle": step.container.Handle(),
				"from":   step.model.From,
				"to":     step.model.To,
				"error":  err,
			},
			"task.handle.download-failed",
		)

		return err
	}

	defer os.Remove(downloadedPath)

	if step.model.Extract {
		extractionDir, err := step.extract(downloadedPath)
		if err != nil {
			return emittable_error.New(err, "Extraction failed")
		}

		defer os.RemoveAll(extractionDir)

		err = step.copyExtractedFiles(extractionDir, step.model.To)
		if err != nil {
			return emittable_error.New(err, "Copying into the container failed")
		}

		return err
	} else {
		reader, writer := io.Pipe()
		defer reader.Close()

		go writeTarTo(filepath.Base(step.model.To), downloadedPath, writer)

		err = step.container.StreamIn(filepath.Dir(step.model.To), reader)
		if err != nil {
			return emittable_error.New(err, "Copying into the container failed")
		}

		return nil
	}
}

func (step *DownloadStep) download() (string, error) {
	url, err := url.ParseRequestURI(step.model.From)
	if err != nil {
		return "", err
	}

	tempFile, err := ioutil.TempFile(step.tempDir, "downloaded")
	if err != nil {
		return "", err
	}
	defer tempFile.Close()

	downloadedFile, err := step.cachedDownloader.Fetch(url, step.model.CacheKey)
	if err != nil {
		return "", err
	}
	defer downloadedFile.Close()

	_, err = io.Copy(tempFile, downloadedFile)
	if err != nil {
		return "", err
	}

	return tempFile.Name(), nil
}

func (step *DownloadStep) extract(downloadedPath string) (string, error) {
	extractionDir, err := ioutil.TempDir(step.tempDir, "extracted")
	if err != nil {
		return "", err
	}

	err = step.extractor.Extract(downloadedPath, extractionDir)
	if err != nil {
		step.logger.Warnd(
			map[string]interface{}{
				"error": err.Error(),
				"url":   step.model.From,
			},
			"downloader.extract-failed",
		)

		return "", err
	}

	return extractionDir, nil
}

func (step *DownloadStep) Cancel() {}

func (step *DownloadStep) Cleanup() {}

func (step *DownloadStep) copyExtractedFiles(source string, destination string) error {
	reader, writer := io.Pipe()

	go func() {
		compressor.WriteTar(source+string(filepath.Separator), writer)
		writer.Close()
	}()
	defer reader.Close()

	return step.container.StreamIn(destination, reader)
}

func writeTarTo(name string, sourcePath string, destination io.WriteCloser) error {
	source, err := os.Open(sourcePath)
	if err != nil {
		return err
	}
	defer source.Close()

	tarWriter := tar.NewWriter(destination)

	fileInfo, err := source.Stat()
	if err != nil {
		return err
	}

	err = tarWriter.WriteHeader(&tar.Header{
		Name:       name,
		Size:       fileInfo.Size(),
		Mode:       0644,
		AccessTime: time.Now(),
		ChangeTime: time.Now(),
	})
	if err != nil {
		return err
	}

	_, err = io.Copy(tarWriter, source)
	if err != nil {
		return err
	}

	if err := tarWriter.Flush(); err != nil {
		return err
	}

	if err := destination.Close(); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package responses

import (
	"encoding/json"
	"fmt"
	"github.com/perthgophers/puddle/messagerouter"
	"io/ioutil"
	"net/http"
	"net/url"
)

type cookiejar struct {
	jar map[string][]*http.Cookie
}

func (p *cookiejar) SetCookies(u *url.URL, cookies []*http.Cookie) {
	fmt.Printf("The URL is %s\n", u.String())
	fmt.Printf("The cookie being set is : %s\n", cookies)
	p.jar[u.Host] = cookies
}

func (p *cookiejar) Cookies(u *url.URL) []*http.Cookie {
	fmt.Printf("The URL is %s\n", u.String())
	fmt.Printf("The cookie being returned is : %s\n", p.jar[u.Host])
	return p.jar[u.Host]
}

func MemeGenerator(cr *messagerouter.CommandRequest, w messagerouter.ResponseWriter) error {
	// Create cookiejar so we can authenticate to imgur API
	// http://stackoverflow.com/questions/11361431/authenticated-http-client-requests-from-golang

	type HTTPResponse struct {
		Data []struct {
			AccountID    int         `json:"account_id"`
			AccountURL   string      `json:"account_url"`
			CommentCount int         `json:"comment_count"`
			Cover        string      `json:"cover"`
			CoverHeight  int         `json:"cover_height"`
			CoverWidth   int         `json:"cover_width"`
			Datetime     int         `json:"datetime"`
			Description  interface{} `json:"description"`
			Downs        int         `json:"downs"`
			Favorite     bool        `json:"favorite"`
			ID           string      `json:"id"`
			ImagesCount  int         `json:"images_count"`
			InGallery    bool        `json:"in_gallery"`
			IsAd         bool        `json:"is_ad"`
			IsAlbum      bool        `json:"is_album"`
			Layout       string      `json:"layout"`
			Link         string      `json:"link"`
			Nsfw         bool        `json:"nsfw"`
			Points       int         `json:"points"`
			Privacy      string      `json:"privacy"`
			Score        int         `json:"score"`
			Section      string      `json:"section"`
			Title        string      `json:"title"`
			Topic        string      `json:"topic"`
			TopicID      int         `json:"topic_id"`
			Ups          int         `json:"ups"`
			Views        int         `json:"views"`
			Vote         interface{} `json:"vote"`
		} `json:"data"`
	}

	client := &http.Client{}
	jar := &cookiejar{}
	jar.jar = make(map[string][]*http.Cookie)
	client.Jar = jar

	// Authenticate
	//req, err := http.NewRequest("GET", "https://api.imgur.com/oauth2/authorize?client_id=91101181cd13628&response_type=token&state=auth", nil)
	req, err := http.NewRequest("GET", "https://api.imgur.com/3/g/memes/", nil)
	req.Header.Set("Authorization", "Client-ID 91101181cd13628")
	resp, err := client.Do(req)

	if err != nil {
		fmt.Println("Unable to get auth token from imgur api")
	}

	defer resp.Body.Close()

	var httpResponse HTTPResponse
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println("Unable to read httpResponse")
	}

	err = json.Unmarshal(b, &httpResponse)

	if err != nil {
		fmt.Println("Unable to unmarshall", err)
	}

	// Post meme to slack channel using RTM
	w.Write(string(httpResponse.Data[0].Link))

	return nil
}

func init() {
	Handle("!meme", MemeGenerator)
}
<commit_msg>Tidied up meme code<commit_after>package responses

import (
	"encoding/json"
	"fmt"
	"github.com/perthgophers/puddle/messagerouter"
	"io/ioutil"
	"net/http"
	"net/url"
)

type cookiejar struct {
	jar map[string][]*http.Cookie
}

// Used to set cookies into HTTP lib cookiejar
func (p *cookiejar) SetCookies(u *url.URL, cookies []*http.Cookie) {
	fmt.Printf("The URL is %s\n", u.String())
	fmt.Printf("The cookie being set is : %s\n", cookies)
	p.jar[u.Host] = cookies
}

// Used to get cookies from HTTP lib cookiejar
func (p *cookiejar) Cookies(u *url.URL) []*http.Cookie {
	fmt.Printf("The URL is %s\n", u.String())
	fmt.Printf("The cookie being returned is : %s\n", p.jar[u.Host])
	return p.jar[u.Host]
}

// MemeGenerator will post the latest meme from imgur meme gallery
func MemeGenerator(cr *messagerouter.CommandRequest, w messagerouter.ResponseWriter) error {
	// Create cookiejar so we can authenticate to imgur API
	// http://stackoverflow.com/questions/11361431/authenticated-http-client-requests-from-golang

	// HTTP object to capture imgur API responses
	type HTTPResponse struct {
		Data []struct {
			AccountID    int         `json:"account_id"`
			AccountURL   string      `json:"account_url"`
			CommentCount int         `json:"comment_count"`
			Cover        string      `json:"cover"`
			CoverHeight  int         `json:"cover_height"`
			CoverWidth   int         `json:"cover_width"`
			Datetime     int         `json:"datetime"`
			Description  interface{} `json:"description"`
			Downs        int         `json:"downs"`
			Favorite     bool        `json:"favorite"`
			ID           string      `json:"id"`
			ImagesCount  int         `json:"images_count"`
			InGallery    bool        `json:"in_gallery"`
			IsAd         bool        `json:"is_ad"`
			IsAlbum      bool        `json:"is_album"`
			Layout       string      `json:"layout"`
			Link         string      `json:"link"`
			Nsfw         bool        `json:"nsfw"`
			Points       int         `json:"points"`
			Privacy      string      `json:"privacy"`
			Score        int         `json:"score"`
			Section      string      `json:"section"`
			Title        string      `json:"title"`
			Topic        string      `json:"topic"`
			TopicID      int         `json:"topic_id"`
			Ups          int         `json:"ups"`
			Views        int         `json:"views"`
			Vote         interface{} `json:"vote"`
		} `json:"data"`
	}

	client := &http.Client{}
	cjar := &cookiejar{}
	cjar.jar = make(map[string][]*http.Cookie)
	client.Jar = cjar

	// Authenticate
	req, err := http.NewRequest("GET", "https://api.imgur.com/3/g/memes/", nil)
	req.Header.Set("Authorization", "Client-ID 91101181cd13628")
	resp, err := client.Do(req)

	if err != nil {
		fmt.Println("Unable to get auth token from imgur api", err)
	}

	defer resp.Body.Close()

	var httpResponse HTTPResponse
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println("Unable to read", err)
	}

	err = json.Unmarshal(b, &httpResponse)

	if err != nil {
		fmt.Println("Unable to unmarshall", err)
	}

	// Post meme to slack channel using RTM
	w.Write(string(httpResponse.Data[0].Link))

	return nil
}

func init() {
	Handle("!meme", MemeGenerator)
}
<|endoftext|>


<commit_before>package packager

import(
	tp "tritium/proto"
	proto "goprotobuf.googlecode.com/hg/proto"
	yaml "launchpad.net/goyaml"
	"io/ioutil"
	"log"
	"strings"
	"fmt"
	"exec"
	linker "tritium/linker"
	"path/filepath"
)

type Package struct {
	loaded []*PackageInfo
	location string
	LoadPath string
	*tp.Package
}

type PackageInfo struct {
	Name string
	Dependencies []string
	Types []string
}

func BuildDefaultPackage(dir string) (*Package) {
	// Terrible directory handling here... has to be executed from Tritium root
	pkg := NewPackage(dir)

	//pkg.Load("base")
	//pkg.Load("node")
	pkg.Load("libxml")
	println("Packages all loaded")

	return pkg
}

func NewPackage(loadPath string) (*Package){
	return &Package{
		Package: &tp.Package{
			Name: proto.String("combined"),
			Functions: make([]*tp.Function, 0),
			Types: make([]*tp.Type, 0),
		},
		loaded: make([]*PackageInfo, 0),
		LoadPath: loadPath,
	}
}

func (pkg *Package)Load(packageName string) {

	location := filepath.Join(pkg.LoadPath, packageName)
	pkg.location = location

	info := readPackageInfoFile(location)

	if len(info.Dependencies) > 0 {

		println("==========\nLoading dependencies:")

		for _, dependency := range(info.Dependencies) {
			pkg.loadPackageDependency(dependency)
		}

		println("done.\n==========")

	}

	fmt.Printf("%v\n", location)

	for _, typeName := range(info.Types) {
		split := strings.Split(typeName, " < ")
		typeObj := &tp.Type{}
		if len(split) == 2 {
			typeName = split[0]
			index := pkg.findTypeIndex(split[1])

			typeObj.Implements = proto.Int32(int32(index))
		}
		typeObj.Name = proto.String(typeName)
		pkg.Types = append(pkg.Types, typeObj)
	}

	pkg.readHeaderFile(location)

	pkg.readPackageDefinitions(location)

	pkg.inheritFunctions()

	println(" -- done\n\n\n\n\n")
}

func (pkg *Package)resolveFunction(fun *tp.Function) {
	linkingContext := linker.NewLinkingContext(pkg.Package)

//	pkg.resolveFunctionDescendants(fun)

	fmt.Printf("\t -- Resolving --\n")
	fmt.Printf("\t\t -- function: %v\n", fun)

	// Re-uses linker's logic to resolve function definitions
	if ( proto.GetBool( fun.BuiltIn ) == false) {
		typeName := proto.GetString(fun.ScopeType)

		if len(typeName) != 0 {
			// When I pass in functions from the inheritance resolver, they're typeId is already set
			fun.ScopeTypeId = pkg.GetProtoTypeId(fun.ScopeType)
			fun.ScopeType = nil
		}

		localScope := make(linker.LocalDef, len(fun.Args))

		//		fun.ReturnTypeId = pkg.GetProtoTypeId(fun.ReturnType)
		for _, arg := range(fun.Args) {
			argTypeName := arg.TypeString
			var argTypeId int

			if argTypeName != nil {
				// Similar deal. Input functions from inheritance resolution already have ids set

				arg.TypeId = pkg.GetProtoTypeId(arg.TypeString)
				//println("Processing %", proto.GetString(arg.Name))
				argTypeId = pkg.GetTypeId(proto.GetString(arg.TypeString))
				arg.TypeString = nil
			} else {
				argTypeId = int( proto.GetInt32(arg.TypeId) )
			}

			localScope[proto.GetString(arg.Name)] = argTypeId
		}

		//fmt.Printf("Some insitruction: %v, %s", fun.Instruction, proto.GetString(fun.Name) )
		scopeTypeId := int(proto.GetInt32(fun.ScopeTypeId))
		fmt.Printf("\t\t -- opening scope type : %v\n", scopeTypeId)
		returnType := linkingContext.ProcessInstructionWithLocalScope(fun.Instruction, scopeTypeId, localScope)
		fun.ReturnTypeId = proto.Int32(int32(returnType))
	}
	pkg.Package.Functions = append(pkg.Package.Functions, fun)
	fmt.Printf("\t\t -- done --\n")
}

func (pkg *Package)inheritFunctions() {
	fmt.Printf("pkg types: %v", pkg.Types)
	for _, function := range(pkg.Functions) {
		pkg.resolveFunctionDescendants(function)
	}
}

// TODO : Make a hash for this
func (pkg *Package)findDescendentType(thisType int32) int {
	for index, someType := range(pkg.Types) {
		if proto.GetInt32(someType.Implements) == thisType {
			//fmt.Printf("=== %v is ancestor of %v ===\n", thisType, someType)
			return index
		}
	}
	return -1
}

// TODO(SJ) : Make this not suck. I think I could make this 50% shorter if I use reflection
// - Also, I'm assuming a single depth level of inheritance. I'd have to run this function n times for n levels
// - Well that should be fine as long as I run it at the end of every package load

func (pkg *Package)resolveFunctionDescendants(fun *tp.Function) {

	// Check if this function contains any types that have descendants
	name := fun.Stub(pkg.Package)
	println("Checking for inheritance on function:", name )

	newFun := &tp.Function{}
	inherit := false

	// Iterate over ScopeType, Arg types, return Type, opens Type

	// ScopeType

	thisTypeId := proto.GetInt32(fun.ScopeTypeId)
	newType := pkg.findDescendentType(thisTypeId)

	if newType != -1 {
		if !inherit {
			fmt.Printf("\t -- ScopeType : Found ancestral type. Cloning function %v\n", proto.GetString( fun.Name ) )
			newFun = fun.Clone()
			// fmt.Printf("\t -- New fun: %v", newFun)
			inherit = true
		}
		println("\t -- Resetting scopeId")
		newFun.ScopeTypeId = proto.Int32( int32( newType ) )
	}

	// ReturnType

	thisTypeId = proto.GetInt32(fun.ReturnTypeId)
	newType = pkg.findDescendentType(thisTypeId)

	if newType != -1 {
		if !inherit {
			fmt.Printf("\t -- ReturnType : Found ancestral type. Cloning function %v\n", proto.GetString( fun.Name ) )
			newFun = fun.Clone()
			// fmt.Printf("\t -- New fun: %v", newFun)
			inherit = true
		}
		println("\t -- Resetting returnId")
		newFun.ReturnTypeId = proto.Int32( int32( newType ) )
	}

	// OpensType

	thisTypeId = proto.GetInt32(fun.OpensTypeId)
	newType = pkg.findDescendentType(thisTypeId)

	if thisTypeId > 0 && newType != -1 {
		// HACK(SJ) : Fix this
		// I exclude base from inheritance
		if !inherit {
			fmt.Printf("\t -- OpensType : Found ancestral type. Cloning function %v\n", proto.GetString( fun.Name ) )
			newFun = fun.Clone()
			// fmt.Printf("\t -- New fun: %v", newFun)
			inherit = true
		}
		println("\t -- Resetting openTypeId")
		newFun.OpensTypeId = proto.Int32( int32( newType ) )
	}

	// Arguments

	for index, arg := range( fun.Args) {
		thisTypeId = proto.GetInt32(arg.TypeId)
		newType = pkg.findDescendentType(thisTypeId)

		if newType != -1 {

			if !inherit {
				fmt.Printf("\t -- ArgType : Found ancestral type. Cloning function %v\n", proto.GetString( fun.Name ) )
				newFun = fun.Clone()
				// fmt.Printf("\t -- New fun: %v", newFun)
				inherit = true
			}
			println("\t -- Resetting argument")
			newFun.Args[index].TypeId = proto.Int32( int32( newType ) )
		}

	}

	fmt.Printf("\t -- Old function: %v\n\t -- New function: %v\n", fun, newFun)

	if inherit {
		pkg.resolveFunction(newFun)
	}

}

func (pkg *Package)readPackageDefinitions(location string) {

	println(" -- reading definitions")

	// Execute the ts2func-ruby script

	package_name := strings.Split(location,"/")[1]
	input_file := filepath.Join(location, "functions.ts")
	output_file := filepath.Join(location, package_name + ".tf")

	println(input_file)

	// Assume that tritium/bin is in $PATH (it will be when you install the gem)
	// -- if you're developing, add $REPOS/tritium/bin to $PATH

	command := exec.Command("ts2func-ruby", "-s", input_file, output_file)

	//fmt.Printf("\n\nExecuting command: \n %v\n", command)

	output, err := command.CombinedOutput()

	if err != nil {
		fmt.Printf("\tFunction conversion output:\n\t %s", output)
		log.Panic(err)
	}

	functions := &tp.FunctionArray{}
	data, err := ioutil.ReadFile(output_file)

	if err != nil {
		println("Failed to read output file.")
		log.Panic(err)
	}

	err = proto.Unmarshal(data, functions)

	if err != nil {
		println("Failed while loading output from ts2func.")
		println(string(output))
		log.Panic(err)
	}

	//fmt.Printf("functions : %v", functions)
	//fmt.Printf("\n\n prelim pkg functions : %v\n", pkg.Package.Functions) */

	//println("Function count before ", len(pkg.Package.Functions))
	for _, function := range(functions.Functions) {
		fmt.Printf("\t -- function: %v", function)
		pkg.resolveFunction(function)
	}
	//fmt.Printf("\n\npkg functions : %v\n", pkg.Package.Functions)
	//println("Function count after ", len(pkg.Package.Functions))
	//pkg.Package.Functions = functions.Functions

	//fmt.Printf("\n\npkg functions : %v\n", pkg.Package.Functions)

}

func (pkg *Package)Marshal() []byte {
	bytes, err := proto.Marshal(pkg.Package)
	if err != nil {
		log.Panic(err)
	}
	return bytes
}

func (pkg *Package)findTypeIndex(name string) int {
	for index, typeObj := range(pkg.Types) {
		if name == proto.GetString(typeObj.Name) {
			return index
		}
	}

	log.Panic("Bad type load order, type", name, "unknown")
	return -1
}

func (pkg *Package)loadPackageDependency(name string) {

	// Try and load the dependency
	// TODO : remove passing location around since I added it to the Package struct

	// TODO : Check for a pre-built package (pre-req is outputting a .tpkg file upon completion of a package load)

	newPath := filepath.Join(pkg.LoadPath, name)
	_, err := ioutil.ReadDir(newPath)

	if err == nil {
		// Directory exists
		pkg.Load(name)
	} else {
		println("Cannot find package at:", newPath)
		log.Panic(err)
	}

}

// Not fully functional. Dang it.
func readPackageInfoFile(location string) (*PackageInfo){
	packageInfo := &PackageInfo{}
	infoFile, err := ioutil.ReadFile(location + "/package.yml");
	if err != nil {
		log.Panic("No package info file found at " + location + "/package.yml")
	}
	yaml.Unmarshal([]byte(infoFile), &packageInfo)
	//fmt.Printf("--- m:\n%v\n\n", packageInfo)
	return packageInfo
}

// Not fully functional. Dang it.
func (pkg *Package)readHeaderFile(location string) {
	headerFile, err := ioutil.ReadFile(location + "/headers.tf");
	if err != nil {
		log.Panic("No header file found at " + location + "/headers.tf")
	}
	headerLines := strings.Split(string(headerFile), "\n")
	for _, line := range(headerLines) {
		if (len(line) > 2) && (line[:2] != "//") {
			function := &tp.Function{Args: make([]*tp.Function_Argument, 0)}
			line = line[6:]
			methodName := strings.Split(line, "(")[0]
			typeId := 0
			back := line[len(methodName)+1:]
			methodizer := strings.Split(methodName, ".")
			if len(methodizer) > 1 {
				typeStr := methodizer[0]
				typeId = pkg.GetTypeId(typeStr)
				methodName = methodizer[1]
			}
			function.Name = proto.String(methodName)
			function.ScopeTypeId = proto.Int32(int32(typeId))

			splat := strings.Split(back, ")")
			argString := splat[0]
			// Process the arguments
			if len(argString) > 0 {
				argList := strings.Split(argString, ",")
				for _, argLine := range(argList) {
					arg := &tp.Function_Argument{}
					typeStr := strings.Split(strings.Trim(argLine, " \t"), " ")[0]
					typeId = pkg.findTypeIndex(typeStr)
					arg.TypeId = proto.Int32(int32(typeId))
					function.Args = append(function.Args, arg)
				}
			}

			back = splat[1]

			hintStrs := strings.Split(back, "//")
			if len(hintStrs) == 2 {
				hints := strings.Split(hintStrs[1], ",")
				if len(hints) >= 1 {
					 function.ReturnTypeId = proto.Int32(int32(pkg.findTypeIndex(hints[0])))
				}
				if len(hints) >= 2 {
					 function.OpensTypeId = proto.Int32(int32(pkg.findTypeIndex(hints[1])))
				}
			}
			function.BuiltIn = proto.Bool(true)

			pkg.Functions = append(pkg.Functions, function)
		}
	}
}

func (pkg *Package)SerializedOutput() {

	bytes, err := proto.Marshal(pkg.Package)
	if err != nil {
		log.Panic(err)
	}
	println(string(bytes))
}
<commit_msg>Handle edge case where implements field is not set<commit_after>package packager

import(
	tp "tritium/proto"
	proto "goprotobuf.googlecode.com/hg/proto"
	yaml "launchpad.net/goyaml"
	"io/ioutil"
	"log"
	"strings"
	"fmt"
	"exec"
	linker "tritium/linker"
	"path/filepath"
)

type Package struct {
	loaded []*PackageInfo
	location string
	LoadPath string
	*tp.Package
}

type PackageInfo struct {
	Name string
	Dependencies []string
	Types []string
}

func BuildDefaultPackage(dir string) (*Package) {
	// Terrible directory handling here... has to be executed from Tritium root
	pkg := NewPackage(dir)

	//pkg.Load("base")
	//pkg.Load("node")
	pkg.Load("libxml")
	println("Packages all loaded")

	return pkg
}

func NewPackage(loadPath string) (*Package){
	return &Package{
		Package: &tp.Package{
			Name: proto.String("combined"),
			Functions: make([]*tp.Function, 0),
			Types: make([]*tp.Type, 0),
		},
		loaded: make([]*PackageInfo, 0),
		LoadPath: loadPath,
	}
}

func (pkg *Package)Load(packageName string) {

	location := filepath.Join(pkg.LoadPath, packageName)
	pkg.location = location

	info := readPackageInfoFile(location)

	if len(info.Dependencies) > 0 {

		println("==========\nLoading dependencies:")

		for _, dependency := range(info.Dependencies) {
			pkg.loadPackageDependency(dependency)
		}

		println("done.\n==========")

	}

	fmt.Printf("%v\n", location)

	for _, typeName := range(info.Types) {
		split := strings.Split(typeName, " < ")
		typeObj := &tp.Type{}
		if len(split) == 2 {
			typeName = split[0]
			index := pkg.findTypeIndex(split[1])

			typeObj.Implements = proto.Int32(int32(index))
		}
		typeObj.Name = proto.String(typeName)
		pkg.Types = append(pkg.Types, typeObj)
	}

	pkg.readHeaderFile(location)

	pkg.readPackageDefinitions(location)

	pkg.inheritFunctions()

	println(" -- done\n\n\n\n\n")
}

func (pkg *Package)resolveFunction(fun *tp.Function) {
	linkingContext := linker.NewLinkingContext(pkg.Package)

//	pkg.resolveFunctionDescendants(fun)

	fmt.Printf("\t -- Resolving --\n")
	fmt.Printf("\t\t -- function: %v\n", fun)

	// Re-uses linker's logic to resolve function definitions
	if ( proto.GetBool( fun.BuiltIn ) == false) {
		typeName := proto.GetString(fun.ScopeType)

		if len(typeName) != 0 {
			// When I pass in functions from the inheritance resolver, they're typeId is already set
			fun.ScopeTypeId = pkg.GetProtoTypeId(fun.ScopeType)
			fun.ScopeType = nil
		}

		localScope := make(linker.LocalDef, len(fun.Args))

		//		fun.ReturnTypeId = pkg.GetProtoTypeId(fun.ReturnType)
		for _, arg := range(fun.Args) {
			argTypeName := arg.TypeString
			var argTypeId int

			if argTypeName != nil {
				// Similar deal. Input functions from inheritance resolution already have ids set

				arg.TypeId = pkg.GetProtoTypeId(arg.TypeString)
				//println("Processing %", proto.GetString(arg.Name))
				argTypeId = pkg.GetTypeId(proto.GetString(arg.TypeString))
				arg.TypeString = nil
			} else {
				argTypeId = int( proto.GetInt32(arg.TypeId) )
			}

			localScope[proto.GetString(arg.Name)] = argTypeId
		}

		//fmt.Printf("Some insitruction: %v, %s", fun.Instruction, proto.GetString(fun.Name) )
		scopeTypeId := int(proto.GetInt32(fun.ScopeTypeId))
		fmt.Printf("\t\t -- opening scope type : %v\n", scopeTypeId)
		returnType := linkingContext.ProcessInstructionWithLocalScope(fun.Instruction, scopeTypeId, localScope)
		fun.ReturnTypeId = proto.Int32(int32(returnType))
	}
	pkg.Package.Functions = append(pkg.Package.Functions, fun)
	fmt.Printf("\t\t -- done --\n")
}

func (pkg *Package)inheritFunctions() {
	fmt.Printf("pkg types: %v", pkg.Types)
	for _, function := range(pkg.Functions) {
		pkg.resolveFunctionDescendants(function)
	}
}

// TODO : Make a hash for this
func (pkg *Package)findDescendentType(thisType int32) int {
	for index, someType := range(pkg.Types) {
		implements := proto.GetInt32(someType.Implements)
		if implements == thisType && implements != 0 {
			// The implements field defaults to 0. Base doesn't implement Base. Text doesn't implement Base
			// TODO(SJ): make the default value -1 so I know if its not set versus something is inheriting from base
			fmt.Printf("=== %v is ancestor of %v === (%v is of type %v and implements : %v)\n", thisType, someType, proto.GetString(someType.Name), index, implements)
			return index
		}
	}
	return -1
}

// TODO(SJ) : Make this not suck. I think I could make this 50% shorter if I use reflection
// - Also, I'm assuming a single depth level of inheritance. I'd have to run this function n times for n levels
// - Well that should be fine as long as I run it at the end of every package load

func (pkg *Package)resolveFunctionDescendants(fun *tp.Function) {

	// Check if this function contains any types that have descendants
	name := fun.Stub(pkg.Package)
	println("Checking for inheritance on function:", name )

	newFun := &tp.Function{}
	inherit := false

	// Iterate over ScopeType, Arg types, return Type, opens Type

	// ScopeType

	thisTypeId := proto.GetInt32(fun.ScopeTypeId)
	newType := pkg.findDescendentType(thisTypeId)

	if newType != -1 {
		if !inherit {
			fmt.Printf("\t -- ScopeType : Found ancestral type. Cloning function %v\n", proto.GetString( fun.Name ) )
			newFun = fun.Clone()
			// fmt.Printf("\t -- New fun: %v", newFun)
			inherit = true
		}
		println("\t -- Resetting scopeId")
		newFun.ScopeTypeId = proto.Int32( int32( newType ) )
	}

	// ReturnType

	thisTypeId = proto.GetInt32(fun.ReturnTypeId)
	newType = pkg.findDescendentType(thisTypeId)

	if newType != -1 {
		if !inherit {
			fmt.Printf("\t -- ReturnType : Found ancestral type. Cloning function %v\n", proto.GetString( fun.Name ) )
			newFun = fun.Clone()
			// fmt.Printf("\t -- New fun: %v", newFun)
			inherit = true
		}
		println("\t -- Resetting returnId")
		newFun.ReturnTypeId = proto.Int32( int32( newType ) )
	}

	// OpensType

	thisTypeId = proto.GetInt32(fun.OpensTypeId)
	newType = pkg.findDescendentType(thisTypeId)

	if newType != -1 {

		if !inherit {
			fmt.Printf("\t -- OpensType : Found ancestral type. Cloning function %v\n", proto.GetString( fun.Name ) )
			newFun = fun.Clone()
			// fmt.Printf("\t -- New fun: %v", newFun)
			inherit = true
		}
		println("\t -- Resetting openTypeId")
		newFun.OpensTypeId = proto.Int32( int32( newType ) )
	}

	// Arguments

	for index, arg := range( fun.Args) {
		thisTypeId = proto.GetInt32(arg.TypeId)
		newType = pkg.findDescendentType(thisTypeId)

		if newType != -1 {

			if !inherit {
				fmt.Printf("\t -- ArgType : Found ancestral type. Cloning function %v\n", proto.GetString( fun.Name ) )
				newFun = fun.Clone()
				// fmt.Printf("\t -- New fun: %v", newFun)
				inherit = true
			}
			println("\t -- Resetting argument")
			newFun.Args[index].TypeId = proto.Int32( int32( newType ) )
		}

	}

	fmt.Printf("\t -- Old function: %v\n\t -- New function: %v\n", fun, newFun)

	if inherit {
		pkg.resolveFunction(newFun)
	}

}

func (pkg *Package)readPackageDefinitions(location string) {

	println(" -- reading definitions")

	// Execute the ts2func-ruby script

	package_name := strings.Split(location,"/")[1]
	input_file := filepath.Join(location, "functions.ts")
	output_file := filepath.Join(location, package_name + ".tf")

	println(input_file)

	// Assume that tritium/bin is in $PATH (it will be when you install the gem)
	// -- if you're developing, add $REPOS/tritium/bin to $PATH

	command := exec.Command("ts2func-ruby", "-s", input_file, output_file)

	//fmt.Printf("\n\nExecuting command: \n %v\n", command)

	output, err := command.CombinedOutput()

	if err != nil {
		fmt.Printf("\tFunction conversion output:\n\t %s", output)
		log.Panic(err)
	}

	functions := &tp.FunctionArray{}
	data, err := ioutil.ReadFile(output_file)

	if err != nil {
		println("Failed to read output file.")
		log.Panic(err)
	}

	err = proto.Unmarshal(data, functions)

	if err != nil {
		println("Failed while loading output from ts2func.")
		println(string(output))
		log.Panic(err)
	}

	//fmt.Printf("functions : %v", functions)
	//fmt.Printf("\n\n prelim pkg functions : %v\n", pkg.Package.Functions) */

	//println("Function count before ", len(pkg.Package.Functions))
	for _, function := range(functions.Functions) {
		fmt.Printf("\t -- function: %v", function)
		pkg.resolveFunction(function)
	}
	//fmt.Printf("\n\npkg functions : %v\n", pkg.Package.Functions)
	//println("Function count after ", len(pkg.Package.Functions))
	//pkg.Package.Functions = functions.Functions

	//fmt.Printf("\n\npkg functions : %v\n", pkg.Package.Functions)

}

func (pkg *Package)Marshal() []byte {
	bytes, err := proto.Marshal(pkg.Package)
	if err != nil {
		log.Panic(err)
	}
	return bytes
}

func (pkg *Package)findTypeIndex(name string) int {
	for index, typeObj := range(pkg.Types) {
		if name == proto.GetString(typeObj.Name) {
			return index
		}
	}

	log.Panic("Bad type load order, type", name, "unknown")
	return -1
}

func (pkg *Package)loadPackageDependency(name string) {

	// Try and load the dependency
	// TODO : remove passing location around since I added it to the Package struct

	// TODO : Check for a pre-built package (pre-req is outputting a .tpkg file upon completion of a package load)

	newPath := filepath.Join(pkg.LoadPath, name)
	_, err := ioutil.ReadDir(newPath)

	if err == nil {
		// Directory exists
		pkg.Load(name)
	} else {
		println("Cannot find package at:", newPath)
		log.Panic(err)
	}

}

// Not fully functional. Dang it.
func readPackageInfoFile(location string) (*PackageInfo){
	packageInfo := &PackageInfo{}
	infoFile, err := ioutil.ReadFile(location + "/package.yml");
	if err != nil {
		log.Panic("No package info file found at " + location + "/package.yml")
	}
	yaml.Unmarshal([]byte(infoFile), &packageInfo)
	//fmt.Printf("--- m:\n%v\n\n", packageInfo)
	return packageInfo
}

// Not fully functional. Dang it.
func (pkg *Package)readHeaderFile(location string) {
	headerFile, err := ioutil.ReadFile(location + "/headers.tf");
	if err != nil {
		log.Panic("No header file found at " + location + "/headers.tf")
	}
	headerLines := strings.Split(string(headerFile), "\n")
	for _, line := range(headerLines) {
		if (len(line) > 2) && (line[:2] != "//") {
			function := &tp.Function{Args: make([]*tp.Function_Argument, 0)}
			line = line[6:]
			methodName := strings.Split(line, "(")[0]
			typeId := 0
			back := line[len(methodName)+1:]
			methodizer := strings.Split(methodName, ".")
			if len(methodizer) > 1 {
				typeStr := methodizer[0]
				typeId = pkg.GetTypeId(typeStr)
				methodName = methodizer[1]
			}
			function.Name = proto.String(methodName)
			function.ScopeTypeId = proto.Int32(int32(typeId))

			splat := strings.Split(back, ")")
			argString := splat[0]
			// Process the arguments
			if len(argString) > 0 {
				argList := strings.Split(argString, ",")
				for _, argLine := range(argList) {
					arg := &tp.Function_Argument{}
					typeStr := strings.Split(strings.Trim(argLine, " \t"), " ")[0]
					typeId = pkg.findTypeIndex(typeStr)
					arg.TypeId = proto.Int32(int32(typeId))
					function.Args = append(function.Args, arg)
				}
			}

			back = splat[1]

			hintStrs := strings.Split(back, "//")
			if len(hintStrs) == 2 {
				hints := strings.Split(hintStrs[1], ",")
				if len(hints) >= 1 {
					 function.ReturnTypeId = proto.Int32(int32(pkg.findTypeIndex(hints[0])))
				}
				if len(hints) >= 2 {
					 function.OpensTypeId = proto.Int32(int32(pkg.findTypeIndex(hints[1])))
				}
			}
			function.BuiltIn = proto.Bool(true)

			pkg.Functions = append(pkg.Functions, function)
		}
	}
}

func (pkg *Package)SerializedOutput() {

	bytes, err := proto.Marshal(pkg.Package)
	if err != nil {
		log.Panic(err)
	}
	println(string(bytes))
}
<|endoftext|>


<commit_before>package packager

import (
	tp "athena/proto"
	proto "goprotobuf.googlecode.com/hg/proto"
	api "tritium/api"
	"strings"
	"path/filepath"
	"log4go"
	"os"
	"time"
	"fmt"
)

type Package struct {
	loaded       []*PackageInfo
	location     string
	LoadPath     string
	FallbackPath string
	OutputFile   string
	Log          log4go.Logger
	*tp.Package
	Options PackageOptions
}

type PackageInfo struct {
	Name         string
	Dependencies []string
	Types        []string
}

var DefaultPackagePath = "packages"

func LoadDefaultPackage(path *string) *Package {
	if path == nil {
		path = &DefaultPackagePath
	}

	return buildPackage(*path, nil)
}

func OutputDefaultPackage(path string) (pkg *Package, newFilePath string) {
	pkg = BuildDefaultPackage()

	_, err := os.Stat(path)

	if err != nil {
		creationErr := os.MkdirAll(path, uint32(0777))
		if creationErr != nil {
			panic("Could not make path(" + path + "). Error:" + creationErr.String())
		}
	}

	_, name := filepath.Split(pkg.OutputFile)
	newOutputFile := filepath.Join(path, name)

	os.Rename(pkg.OutputFile, newOutputFile)

	return pkg, newOutputFile
}

func BuildDefaultPackage() *Package {
	options := BuildOptions()
	return buildPackage(DefaultPackagePath, options)
}

func buildPackage(path string, options PackageOptions) *Package {
	// Terrible directory handling here... has to be executed from Tritium root

	pkg := NewPackage(path, options)
	rootName := "libxml"

	pkg.Name = proto.String(rootName)
	pkg.Load(rootName)

	return pkg
}

func mergeOptions(options PackageOptions) PackageOptions {
	defaults := fetchDefaultOptions()

	if options == nil {
		return defaults
	}

	for k, _ := range defaults {
		_, ok := options[k]

		if !ok {
			options[k] = defaults[k]
		}
	}

	return options
}

func NewPackage(loadPath string, options PackageOptions) *Package {
	options = mergeOptions(options)

	return &Package{
		Package: &tp.Package{
			Name:         proto.String("combined"),
			Functions:    make([]*tp.Function, 0),
			Types:        make([]*tp.Type, 0),
			Dependencies: make([]string, 0),
		},
		loaded:   make([]*PackageInfo, 0),
		Log:      newLog(),
		LoadPath: loadPath,
		Options:  options,
	}
}

func (pkg *Package) BuildUserPackage(loadPath *string, fallbackPath *string) {
	userPackage := NewUserPackage(loadPath, fallbackPath)
	pkg.Merge(userPackage.Package)
}

func NewUserPackage(loadPath *string, fallbackPath *string) *Package {
	//TODO : Check for user-defined feature support

	userPackage := NewPackage(*loadPath, PackageOptions{"stdout": false, "output_tpkg": false, "use_tpkg": false})

	userPackage.FallbackPath = *fallbackPath

	userPackages, _ := filepath.Glob(filepath.Join(userPackage.LoadPath, "*"))

	for _, path := range userPackages {
		components := strings.Split(path, "/")
		name := components[len(components)-1]
		userPackage.Load(name)
	}

	return userPackage
}

func newLog() log4go.Logger {
	pkgLog := make(log4go.Logger)
	os.Mkdir("tmp", uint32(0777))

	pkgLog.AddFilter("file", log4go.FINE, log4go.NewFileLogWriter("tmp/packager.log", false))
	return pkgLog
}

func (pkg *Package) Load(packageName string) {

	user := api.FetchSessionUser()
	approved := user.RequestFeature("package:" + packageName)

	if !approved {
		panic("Package " + packageName + " not approved for use.")
	}

	err := pkg.loadFromPath(pkg.LoadPath, packageName)

	if err != nil && len(pkg.FallbackPath) != 0 {
		err = pkg.loadFromPath(pkg.FallbackPath, packageName)
	}

	if err != nil {
		panic(*err)
	}

}

func (pkg *Package) loadFromPath(path string, name string) (err *string) {
	pkg.Println(path + ":" + name)
	pkg.Log.Info("\n\n\n\nLoading:%v", path+":"+name)

	loaded := pkg.loadedDependency(name)
	if loaded {
		return
	}

	if pkg.Options["use_tpkg"] {
		err := pkg.open(path, name)
		return err
	}

	location := filepath.Join(path, name)

	old_location := pkg.location
	pkg.location = location

	s := time.Nanoseconds()
	info, err := ReadPackageInfoFile(location)
	f := time.Nanoseconds()
	d := float64(f-s) / 1000.0 / 1000.0 / 1000.0
	fmt.Printf("Time to read package info: %0.6fs\n", d)

	if err != nil {
		return err
	}

	s = time.Nanoseconds()

	if len(info.Dependencies) > 0 {

		for _, dependency := range info.Dependencies {
			pkg.loadPackageDependency(dependency)
		}

	}

	f = time.Nanoseconds()
	d = float64(f-s) / 1000.0 / 1000.0 / 1000.0
	fmt.Printf("Time to load dependencies: %0.6fs\n", d)

	s = time.Nanoseconds()
	for _, typeName := range info.Types {
		split := strings.Split(typeName, " < ")
		typeObj := &tp.Type{}
		if len(split) == 2 {
			typeName = split[0]
			index := pkg.findTypeIndex(split[1])

			typeObj.Implements = proto.Int32(int32(index))
		}
		typeObj.Name = proto.String(typeName)
		pkg.Types = append(pkg.Types, typeObj)
	}
	f = time.Nanoseconds()
	d = float64(f-s) / 1000.0 / 1000.0 / 1000.0
	fmt.Printf("Time to resolve types: %0.6fs\n", d)

	s = time.Nanoseconds()
	pkg.readHeaderFile(location)
	f = time.Nanoseconds()
	d = float64(f-s) / 1000.0 / 1000.0 / 1000.0
	fmt.Printf("Time to load dependencies: %0.6fs\n", d)

	s = time.Nanoseconds()
	entryPoint := filepath.Join(location, "functions.ts")
	ReadPackageDefinitions(pkg.Package, entryPoint)
	f = time.Nanoseconds()
	d = float64(f-s) / 1000.0 / 1000.0 / 1000.0
	fmt.Printf("Time to load dependencies: %0.6fs\n", d)

	s = time.Nanoseconds()
	pkg.inheritFunctions()
	f = time.Nanoseconds()
	d = float64(f-s) / 1000.0 / 1000.0 / 1000.0
	fmt.Printf("Time to load dependencies: %0.6fs\n", d)

	if pkg.Options["output_tpkg"] {
		pkg.write()
	}

	pkg.Println(" -- done")
	pkg.Log.Close()

	// TODO(SJ) : Kind of lame. Ideally I think we load other packages as whole packages and write a *.Merge method
	pkg.location = old_location

	return nil
}
<commit_msg>use new log4go<commit_after>package packager

import (
	tp "athena/proto"
	proto "goprotobuf.googlecode.com/hg/proto"
	api "tritium/api"
	"strings"
	"path/filepath"
	"log4go"
	"os"
	"time"
	"fmt"
)

type Package struct {
	loaded       []*PackageInfo
	location     string
	LoadPath     string
	FallbackPath string
	OutputFile   string
	Log          log4go.Logger
	*tp.Package
	Options PackageOptions
}

type PackageInfo struct {
	Name         string
	Dependencies []string
	Types        []string
}

var DefaultPackagePath = "packages"

func LoadDefaultPackage(path *string) *Package {
	if path == nil {
		path = &DefaultPackagePath
	}

	return buildPackage(*path, nil)
}

func OutputDefaultPackage(path string) (pkg *Package, newFilePath string) {
	pkg = BuildDefaultPackage()

	_, err := os.Stat(path)

	if err != nil {
		creationErr := os.MkdirAll(path, uint32(0777))
		if creationErr != nil {
			panic("Could not make path(" + path + "). Error:" + creationErr.String())
		}
	}

	_, name := filepath.Split(pkg.OutputFile)
	newOutputFile := filepath.Join(path, name)

	os.Rename(pkg.OutputFile, newOutputFile)

	return pkg, newOutputFile
}

func BuildDefaultPackage() *Package {
	options := BuildOptions()
	return buildPackage(DefaultPackagePath, options)
}

func buildPackage(path string, options PackageOptions) *Package {
	// Terrible directory handling here... has to be executed from Tritium root

	pkg := NewPackage(path, options)
	rootName := "libxml"

	pkg.Name = proto.String(rootName)
	pkg.Load(rootName)

	return pkg
}

func mergeOptions(options PackageOptions) PackageOptions {
	defaults := fetchDefaultOptions()

	if options == nil {
		return defaults
	}

	for k, _ := range defaults {
		_, ok := options[k]

		if !ok {
			options[k] = defaults[k]
		}
	}

	return options
}

func NewPackage(loadPath string, options PackageOptions) *Package {
	options = mergeOptions(options)

	return &Package{
		Package: &tp.Package{
			Name:         proto.String("combined"),
			Functions:    make([]*tp.Function, 0),
			Types:        make([]*tp.Type, 0),
			Dependencies: make([]string, 0),
		},
		loaded:   make([]*PackageInfo, 0),
		Log:      newLog(),
		LoadPath: loadPath,
		Options:  options,
	}
}

func (pkg *Package) BuildUserPackage(loadPath *string, fallbackPath *string) {
	userPackage := NewUserPackage(loadPath, fallbackPath)
	pkg.Merge(userPackage.Package)
}

func NewUserPackage(loadPath *string, fallbackPath *string) *Package {
	//TODO : Check for user-defined feature support

	userPackage := NewPackage(*loadPath, PackageOptions{"stdout": false, "output_tpkg": false, "use_tpkg": false})

	userPackage.FallbackPath = *fallbackPath

	userPackages, _ := filepath.Glob(filepath.Join(userPackage.LoadPath, "*"))

	for _, path := range userPackages {
		components := strings.Split(path, "/")
		name := components[len(components)-1]
		userPackage.Load(name)
	}

	return userPackage
}

func newLog() log4go.Logger {
	pkgLog := make(log4go.Logger)
	os.Mkdir("tmp", uint32(0777))

	pkgLog.AddFilter("file", log4go.DEBUG, log4go.NewFileLogWriter("tmp/packager.log", false))
	return pkgLog
}

func (pkg *Package) Load(packageName string) {

	user := api.FetchSessionUser()
	approved := user.RequestFeature("package:" + packageName)

	if !approved {
		panic("Package " + packageName + " not approved for use.")
	}

	err := pkg.loadFromPath(pkg.LoadPath, packageName)

	if err != nil && len(pkg.FallbackPath) != 0 {
		err = pkg.loadFromPath(pkg.FallbackPath, packageName)
	}

	if err != nil {
		panic(*err)
	}

}

func (pkg *Package) loadFromPath(path string, name string) (err *string) {
	pkg.Println(path + ":" + name)
	pkg.Log.Info("\n\n\n\nLoading:%v", path+":"+name)

	loaded := pkg.loadedDependency(name)
	if loaded {
		return
	}

	if pkg.Options["use_tpkg"] {
		err := pkg.open(path, name)
		return err
	}

	location := filepath.Join(path, name)

	old_location := pkg.location
	pkg.location = location

	s := time.Nanoseconds()
	info, err := ReadPackageInfoFile(location)
	f := time.Nanoseconds()
	d := float64(f-s) / 1000.0 / 1000.0 / 1000.0
	fmt.Printf("Time to read package info: %0.6fs\n", d)

	if err != nil {
		return err
	}

	s = time.Nanoseconds()

	if len(info.Dependencies) > 0 {

		for _, dependency := range info.Dependencies {
			pkg.loadPackageDependency(dependency)
		}

	}

	f = time.Nanoseconds()
	d = float64(f-s) / 1000.0 / 1000.0 / 1000.0
	fmt.Printf("Time to load dependencies: %0.6fs\n", d)

	s = time.Nanoseconds()
	for _, typeName := range info.Types {
		split := strings.Split(typeName, " < ")
		typeObj := &tp.Type{}
		if len(split) == 2 {
			typeName = split[0]
			index := pkg.findTypeIndex(split[1])

			typeObj.Implements = proto.Int32(int32(index))
		}
		typeObj.Name = proto.String(typeName)
		pkg.Types = append(pkg.Types, typeObj)
	}
	f = time.Nanoseconds()
	d = float64(f-s) / 1000.0 / 1000.0 / 1000.0
	fmt.Printf("Time to resolve types: %0.6fs\n", d)

	s = time.Nanoseconds()
	pkg.readHeaderFile(location)
	f = time.Nanoseconds()
	d = float64(f-s) / 1000.0 / 1000.0 / 1000.0
	fmt.Printf("Time to load dependencies: %0.6fs\n", d)

	s = time.Nanoseconds()
	entryPoint := filepath.Join(location, "functions.ts")
	ReadPackageDefinitions(pkg.Package, entryPoint)
	f = time.Nanoseconds()
	d = float64(f-s) / 1000.0 / 1000.0 / 1000.0
	fmt.Printf("Time to load dependencies: %0.6fs\n", d)

	s = time.Nanoseconds()
	pkg.inheritFunctions()
	f = time.Nanoseconds()
	d = float64(f-s) / 1000.0 / 1000.0 / 1000.0
	fmt.Printf("Time to load dependencies: %0.6fs\n", d)

	if pkg.Options["output_tpkg"] {
		pkg.write()
	}

	pkg.Println(" -- done")
	pkg.Log.Close()

	// TODO(SJ) : Kind of lame. Ideally I think we load other packages as whole packages and write a *.Merge method
	pkg.location = old_location

	return nil
}
<|endoftext|>


<commit_before>package parser

// Type tags so we know what kind of token we have
type lexeme int
const (
  LPAREN = iota
  RPAREN
  LBRACE
  RBRACE
  COMMA
  EQUAL
  STRING
  REGEXP
  GVAR
  LVAR
  KWD
  ID
  FUNC
  TYPE
  IMPORT
  READ
  EOF
)
// Map the numeric tags back to names for debugging purposes.
var lexemeName [17]string

// A token has a type (aka lexeme) and an actual value
type token struct {
  Lexeme lexeme
  Value string
}

func init() {
  lexemeName[LPAREN] = "LPAREN"
  lexemeName[RPAREN] = "RPAREN"
  lexemeName[LBRACE] = "LBRACE"
  lexemeName[RBRACE] = "RBRACE"
  lexemeName[COMMA]  = "COMMA"
  lexemeName[EQUAL]  = "EQUAL"
  lexemeName[STRING] = "STRING"
  lexemeName[REGEXP] = "REGEXP"
  lexemeName[GVAR]   = "GVAR"
  lexemeName[LVAR]   = "LVAR"
  lexemeName[KWD]    = "KWD"
  lexemeName[ID]     = "ID"
  lexemeName[FUNC]   = "FUNC"
  lexemeName[TYPE]   = "TYPE"
  lexemeName[IMPORT] = "IMPORT"
  lexemeName[READ]   = "READ"
  lexemeName[EOF]    = "EOF"
}<commit_msg>Helper functions for discarding whitespace and comments.<commit_after>package parser

// Type tags so we know what kind of token we have
type lexeme int
const (
  LPAREN = iota
  RPAREN
  LBRACE
  RBRACE
  COMMA
  EQUAL
  STRING
  REGEXP
  POS
  GVAR
  LVAR
  KWD
  ID
  FUNC
  TYPE
  IMPORT
  READ
  EOF
)
// Map the numeric tags back to names for debugging purposes.
var lexemeName [18]string

// A token has a type (aka lexeme), a value, and a line number
type token struct {
  Lexeme lexeme
  Value string
  LineNum int
}

/*
  Represent a tokenizer with an stuct containing the remaining source text
  and the line number. Easier than using a stateless tokenizing function that
  returns them as extra values and requires the parser to keep track of them.
*/
type tokenizer struct {
  Source []byte
  LineNum int
}

// Discard leading spaces (excluding newlines) in the source text.
func (t *tokenizer) discardSpaces() {
  var i int
  var c byte
  for i, c = 0, t.Source[0]; c == ' ' || c == '\t'; i, c = i+1, t.Source[i+1] { }
  t.Source = t.Source[i:]
}

// Discard leading text until a newline is found.
func(t *tokenizer) discardLine() {
  var i int
  var c byte
  for i, c = 0, t.Source[0]; c != '\n'; i, c = i+1, t.Source[i+1] { }
  t.Source = t.Source[i:]
}

// Discard leading comments in the source text.
func (t *tokenizer) discardComments() {
  switch t.Source[0] {
  case '#':
    t.discardLine()
  case '/':
    switch t.Source[1] {
    case '/':
      t.discardLine()
    case '*':
      t.discardBlockComment()
    }
  }
}

// Helper for discarding block comments.
// TO DO: ERROR HANDLING FOR UNTERMINATED COMMENTS
func (t *tokenizer) discardBlockComment() {
  depth, i, length := 1, 2, len(t.Source)
  for depth > 0 {
    if i >= length {
      // ERROR
    }
    switch t.Source[i] {
    case '\n':
      t.LineNum++
    case '/':
      i++
      if i >= length {
        // ERROR
      }
      if t.Source[i] == '*' {
        depth++
      }
    case '*':
      i++
      if i >= length {
        // ERROR
      }
      if t.Source[i] == '/' {
        depth--
      }
    }
    i++
  }
  t.Source = t.Source[i:]
}

func init() {
  // Is there a more elegant way to do this?
  lexemeName[LPAREN] = "LPAREN"
  lexemeName[RPAREN] = "RPAREN"
  lexemeName[LBRACE] = "LBRACE"
  lexemeName[RBRACE] = "RBRACE"
  lexemeName[COMMA]  = "COMMA"
  lexemeName[EQUAL]  = "EQUAL"
  lexemeName[STRING] = "STRING"
  lexemeName[REGEXP] = "REGEXP"
  lexemeName[POS]    = "POS"
  lexemeName[GVAR]   = "GVAR"
  lexemeName[LVAR]   = "LVAR"
  lexemeName[KWD]    = "KWD"
  lexemeName[ID]     = "ID"
  lexemeName[FUNC]   = "FUNC"
  lexemeName[TYPE]   = "TYPE"
  lexemeName[IMPORT] = "IMPORT"
  lexemeName[READ]   = "READ"
  lexemeName[EOF]    = "EOF"
}
<|endoftext|>