<commit_before>package libnetwork

import (
	"fmt"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/libkv/store/boltdb"
	"github.com/docker/libkv/store/consul"
	"github.com/docker/libkv/store/etcd"
	"github.com/docker/libkv/store/zookeeper"
	"github.com/docker/libnetwork/datastore"
)

func registerKVStores() {
	consul.Register()
	zookeeper.Register()
	etcd.Register()
	boltdb.Register()
}

func (c *controller) initScopedStore(scope string, scfg *datastore.ScopeCfg) error {
	store, err := datastore.NewDataStore(scope, scfg)
	if err != nil {
		return err
	}
	c.Lock()
	c.stores = append(c.stores, store)
	c.Unlock()

	return nil
}

func (c *controller) initStores() error {
	registerKVStores()

	c.Lock()
	if c.cfg == nil {
		c.Unlock()
		return nil
	}
	scopeConfigs := c.cfg.Scopes
	c.stores = nil
	c.Unlock()

	for scope, scfg := range scopeConfigs {
		if err := c.initScopedStore(scope, scfg); err != nil {
			return err
		}
	}

	c.startWatch()
	return nil
}

func (c *controller) closeStores() {
	for _, store := range c.getStores() {
		store.Close()
	}
}

func (c *controller) getStore(scope string) datastore.DataStore {
	c.Lock()
	defer c.Unlock()

	for _, store := range c.stores {
		if store.Scope() == scope {
			return store
		}
	}

	return nil
}

func (c *controller) getStores() []datastore.DataStore {
	c.Lock()
	defer c.Unlock()

	return c.stores
}

func (c *controller) getNetworkFromStore(nid string) (*network, error) {
	for _, store := range c.getStores() {
		n := &network{id: nid, ctrlr: c}
		err := store.GetObject(datastore.Key(n.Key()...), n)
		// Continue searching in the next store if the key is not found in this store
		if err != nil {
			if err != datastore.ErrKeyNotFound {
				log.Debugf("could not find network %s: %v", nid, err)
			}
			continue
		}

		ec := &endpointCnt{n: n}
		err = store.GetObject(datastore.Key(ec.Key()...), ec)
		if err != nil && !n.inDelete {
			return nil, fmt.Errorf("could not find endpoint count for network %s: %v", n.Name(), err)
		}

		n.epCnt = ec
		n.scope = store.Scope()
		return n, nil
	}

	return nil, fmt.Errorf("network %s not found", nid)
}

func (c *controller) getNetworksForScope(scope string) ([]*network, error) {
	var nl []*network

	store := c.getStore(scope)
	if store == nil {
		return nil, nil
	}

	kvol, err := store.List(datastore.Key(datastore.NetworkKeyPrefix),
		&network{ctrlr: c})
	if err != nil && err != datastore.ErrKeyNotFound {
		return nil, fmt.Errorf("failed to get networks for scope %s: %v",
			scope, err)
	}

	for _, kvo := range kvol {
		n := kvo.(*network)
		n.ctrlr = c

		ec := &endpointCnt{n: n}
		err = store.GetObject(datastore.Key(ec.Key()...), ec)
		if err != nil && !n.inDelete {
			log.Warnf("Could not find endpoint count key %s for network %s while listing: %v", datastore.Key(ec.Key()...), n.Name(), err)
			continue
		}

		n.epCnt = ec
		n.scope = scope
		nl = append(nl, n)
	}

	return nl, nil
}

func (c *controller) getNetworksFromStore() ([]*network, error) {
	var nl []*network

	for _, store := range c.getStores() {
		kvol, err := store.List(datastore.Key(datastore.NetworkKeyPrefix),
			&network{ctrlr: c})
		// Continue searching in the next store if no keys found in this store
		if err != nil {
			if err != datastore.ErrKeyNotFound {
				log.Debugf("failed to get networks for scope %s: %v", store.Scope(), err)
			}
			continue
		}

		for _, kvo := range kvol {
			n := kvo.(*network)
			n.Lock()
			n.ctrlr = c
			n.Unlock()

			ec := &endpointCnt{n: n}
			err = store.GetObject(datastore.Key(ec.Key()...), ec)
			if err != nil && !n.inDelete {
				log.Warnf("could not find endpoint count key %s for network %s while listing: %v", datastore.Key(ec.Key()...), n.Name(), err)
				continue
			}

			n.Lock()
			n.epCnt = ec
			n.scope = store.Scope()
			n.Unlock()
			nl = append(nl, n)
		}
	}

	return nl, nil
}

func (n *network) getEndpointFromStore(eid string) (*endpoint, error) {
	var errors []string
	for _, store := range n.ctrlr.getStores() {
		ep := &endpoint{id: eid, network: n}
		err := store.GetObject(datastore.Key(ep.Key()...), ep)
		// Continue searching in the next store if the key is not found in this store
		if err != nil {
			if err != datastore.ErrKeyNotFound {
				errors = append(errors, fmt.Sprintf("{%s:%v}, ", store.Scope(), err))
				log.Debugf("could not find endpoint %s in %s: %v", eid, store.Scope(), err)
			}
			continue
		}
		return ep, nil
	}
	return nil, fmt.Errorf("could not find endpoint %s: %v", eid, errors)
}

func (n *network) getEndpointsFromStore() ([]*endpoint, error) {
	var epl []*endpoint

	tmp := endpoint{network: n}
	for _, store := range n.getController().getStores() {
		kvol, err := store.List(datastore.Key(tmp.KeyPrefix()...), &endpoint{network: n})
		// Continue searching in the next store if no keys found in this store
		if err != nil {
			if err != datastore.ErrKeyNotFound {
				log.Debugf("failed to get endpoints for network %s scope %s: %v",
					n.Name(), store.Scope(), err)
			}
			continue
		}

		for _, kvo := range kvol {
			ep := kvo.(*endpoint)
			epl = append(epl, ep)
		}
	}

	return epl, nil
}

func (c *controller) updateToStore(kvObject datastore.KVObject) error {
	cs := c.getStore(kvObject.DataScope())
	if cs == nil {
		return fmt.Errorf("datastore for scope %q is not initialized ", kvObject.DataScope())
	}

	if err := cs.PutObjectAtomic(kvObject); err != nil {
		if err == datastore.ErrKeyModified {
			return err
		}
		return fmt.Errorf("failed to update store for object type %T: %v", kvObject, err)
	}

	return nil
}

func (c *controller) deleteFromStore(kvObject datastore.KVObject) error {
	cs := c.getStore(kvObject.DataScope())
	if cs == nil {
		return fmt.Errorf("datastore for scope %q is not initialized ", kvObject.DataScope())
	}

retry:
	if err := cs.DeleteObjectAtomic(kvObject); err != nil {
		if err == datastore.ErrKeyModified {
			if err := cs.GetObject(datastore.Key(kvObject.Key()...), kvObject); err != nil {
				return fmt.Errorf("could not update the kvobject to latest when trying to delete: %v", err)
			}
			goto retry
		}
		return err
	}

	return nil
}

type netWatch struct {
	localEps  map[string]*endpoint
	remoteEps map[string]*endpoint
	stopCh    chan struct{}
}

func (c *controller) getLocalEps(nw *netWatch) []*endpoint {
	c.Lock()
	defer c.Unlock()

	var epl []*endpoint
	for _, ep := range nw.localEps {
		epl = append(epl, ep)
	}

	return epl
}

func (c *controller) watchSvcRecord(ep *endpoint) {
	c.watchCh <- ep
}

func (c *controller) unWatchSvcRecord(ep *endpoint) {
	c.unWatchCh <- ep
}

func (c *controller) networkWatchLoop(nw *netWatch, ep *endpoint, ecCh <-chan datastore.KVObject) {
	for {
		select {
		case <-nw.stopCh:
			return
		case o := <-ecCh:
			ec := o.(*endpointCnt)

			epl, err := ec.n.getEndpointsFromStore()
			if err != nil {
				break
			}

			c.Lock()
			var addEp []*endpoint

			delEpMap := make(map[string]*endpoint)
			renameEpMap := make(map[string]bool)
			for k, v := range nw.remoteEps {
				delEpMap[k] = v
			}

			for _, lEp := range epl {
				if _, ok := nw.localEps[lEp.ID()]; ok {
					continue
				}

				if ep, ok := nw.remoteEps[lEp.ID()]; ok {
					// On a container rename EP ID will remain
					// the same but the name will change. service
					// records should reflect the change.
					// Keep old EP entry in the delEpMap and add
					// EP from the store (which has the new name)
					// into the new list
					if lEp.name == ep.name {
						delete(delEpMap, lEp.ID())
						continue
					}
					renameEpMap[lEp.ID()] = true
				}
				nw.remoteEps[lEp.ID()] = lEp
				addEp = append(addEp, lEp)
			}

			// EPs whose name are to be deleted from the svc records
			// should also be removed from nw's remote EP list, except
			// the ones that are getting renamed.
			for _, lEp := range delEpMap {
				if !renameEpMap[lEp.ID()] {
					delete(nw.remoteEps, lEp.ID())
				}
			}
			c.Unlock()

			for _, lEp := range delEpMap {
				ep.getNetwork().updateSvcRecord(lEp, c.getLocalEps(nw), false)

			}
			for _, lEp := range addEp {
				ep.getNetwork().updateSvcRecord(lEp, c.getLocalEps(nw), true)
			}
		}
	}
}

func (c *controller) processEndpointCreate(nmap map[string]*netWatch, ep *endpoint) {
	c.Lock()
	nw, ok := nmap[ep.getNetwork().ID()]
	c.Unlock()

	if ok {
		// Update the svc db for the local endpoint join right away
		ep.getNetwork().updateSvcRecord(ep, c.getLocalEps(nw), true)

		c.Lock()
		nw.localEps[ep.ID()] = ep

		// If we had learned that from the kv store remove it
		// from remote ep list now that we know that this is
		// indeed a local endpoint
		delete(nw.remoteEps, ep.ID())
		c.Unlock()
		return
	}

	nw = &netWatch{
		localEps:  make(map[string]*endpoint),
		remoteEps: make(map[string]*endpoint),
	}

	// Update the svc db for the local endpoint join right away
	// Do this before adding this ep to localEps so that we don't
	// try to update this ep's container's svc records
	ep.getNetwork().updateSvcRecord(ep, c.getLocalEps(nw), true)

	c.Lock()
	nw.localEps[ep.ID()] = ep
	nmap[ep.getNetwork().ID()] = nw
	nw.stopCh = make(chan struct{})
	c.Unlock()

	store := c.getStore(ep.getNetwork().DataScope())
	if store == nil {
		return
	}

	if !store.Watchable() {
		return
	}

	ch, err := store.Watch(ep.getNetwork().getEpCnt(), nw.stopCh)
	if err != nil {
		log.Warnf("Error creating watch for network: %v", err)
		return
	}

	go c.networkWatchLoop(nw, ep, ch)
}

func (c *controller) processEndpointDelete(nmap map[string]*netWatch, ep *endpoint) {
	c.Lock()
	nw, ok := nmap[ep.getNetwork().ID()]

	if ok {
		delete(nw.localEps, ep.ID())
		c.Unlock()

		// Update the svc db about local endpoint leave right away
		// Do this after we remove this ep from localEps so that we
		// don't try to remove this svc record from this ep's container.
		ep.getNetwork().updateSvcRecord(ep, c.getLocalEps(nw), false)

		c.Lock()
		if len(nw.localEps) == 0 {
			close(nw.stopCh)

			// This is the last container going away for the network. Destroy
			// this network's svc db entry
			delete(c.svcRecords, ep.getNetwork().ID())

			delete(nmap, ep.getNetwork().ID())
		}
	}
	c.Unlock()
}

func (c *controller) watchLoop() {
	for {
		select {
		case ep := <-c.watchCh:
			c.processEndpointCreate(c.nmap, ep)
		case ep := <-c.unWatchCh:
			c.processEndpointDelete(c.nmap, ep)
		}
	}
}

func (c *controller) startWatch() {
	if c.watchCh != nil {
		return
	}
	c.watchCh = make(chan *endpoint)
	c.unWatchCh = make(chan *endpoint)
	c.nmap = make(map[string]*netWatch)

	go c.watchLoop()
}

func (c *controller) networkCleanup() {
	networks, err := c.getNetworksFromStore()
	if err != nil {
		log.Warnf("Could not retrieve networks from store(s) during network cleanup: %v", err)
		return
	}

	for _, n := range networks {
		if n.inDelete {
			log.Infof("Removing stale network %s (%s)", n.Name(), n.ID())
			if err := n.delete(true); err != nil {
				log.Debugf("Error while removing stale network: %v", err)
			}
		}
	}
}

var populateSpecial NetworkWalker = func(nw Network) bool {
	if n := nw.(*network); n.hasSpecialDriver() {
		if err := n.getController().addNetwork(n); err != nil {
			log.Warnf("Failed to populate network %q with driver %q", nw.Name(), nw.Type())
		}
	}
	return false
}
<commit_msg>Avoid k/v store endpoint update in swarm mode<commit_after>package libnetwork

import (
	"fmt"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/libkv/store/boltdb"
	"github.com/docker/libkv/store/consul"
	"github.com/docker/libkv/store/etcd"
	"github.com/docker/libkv/store/zookeeper"
	"github.com/docker/libnetwork/datastore"
)

func registerKVStores() {
	consul.Register()
	zookeeper.Register()
	etcd.Register()
	boltdb.Register()
}

func (c *controller) initScopedStore(scope string, scfg *datastore.ScopeCfg) error {
	store, err := datastore.NewDataStore(scope, scfg)
	if err != nil {
		return err
	}
	c.Lock()
	c.stores = append(c.stores, store)
	c.Unlock()

	return nil
}

func (c *controller) initStores() error {
	registerKVStores()

	c.Lock()
	if c.cfg == nil {
		c.Unlock()
		return nil
	}
	scopeConfigs := c.cfg.Scopes
	c.stores = nil
	c.Unlock()

	for scope, scfg := range scopeConfigs {
		if err := c.initScopedStore(scope, scfg); err != nil {
			return err
		}
	}

	c.startWatch()
	return nil
}

func (c *controller) closeStores() {
	for _, store := range c.getStores() {
		store.Close()
	}
}

func (c *controller) getStore(scope string) datastore.DataStore {
	c.Lock()
	defer c.Unlock()

	for _, store := range c.stores {
		if store.Scope() == scope {
			return store
		}
	}

	return nil
}

func (c *controller) getStores() []datastore.DataStore {
	c.Lock()
	defer c.Unlock()

	return c.stores
}

func (c *controller) getNetworkFromStore(nid string) (*network, error) {
	for _, store := range c.getStores() {
		n := &network{id: nid, ctrlr: c}
		err := store.GetObject(datastore.Key(n.Key()...), n)
		// Continue searching in the next store if the key is not found in this store
		if err != nil {
			if err != datastore.ErrKeyNotFound {
				log.Debugf("could not find network %s: %v", nid, err)
			}
			continue
		}

		ec := &endpointCnt{n: n}
		err = store.GetObject(datastore.Key(ec.Key()...), ec)
		if err != nil && !n.inDelete {
			return nil, fmt.Errorf("could not find endpoint count for network %s: %v", n.Name(), err)
		}

		n.epCnt = ec
		n.scope = store.Scope()
		return n, nil
	}

	return nil, fmt.Errorf("network %s not found", nid)
}

func (c *controller) getNetworksForScope(scope string) ([]*network, error) {
	var nl []*network

	store := c.getStore(scope)
	if store == nil {
		return nil, nil
	}

	kvol, err := store.List(datastore.Key(datastore.NetworkKeyPrefix),
		&network{ctrlr: c})
	if err != nil && err != datastore.ErrKeyNotFound {
		return nil, fmt.Errorf("failed to get networks for scope %s: %v",
			scope, err)
	}

	for _, kvo := range kvol {
		n := kvo.(*network)
		n.ctrlr = c

		ec := &endpointCnt{n: n}
		err = store.GetObject(datastore.Key(ec.Key()...), ec)
		if err != nil && !n.inDelete {
			log.Warnf("Could not find endpoint count key %s for network %s while listing: %v", datastore.Key(ec.Key()...), n.Name(), err)
			continue
		}

		n.epCnt = ec
		n.scope = scope
		nl = append(nl, n)
	}

	return nl, nil
}

func (c *controller) getNetworksFromStore() ([]*network, error) {
	var nl []*network

	for _, store := range c.getStores() {
		kvol, err := store.List(datastore.Key(datastore.NetworkKeyPrefix),
			&network{ctrlr: c})
		// Continue searching in the next store if no keys found in this store
		if err != nil {
			if err != datastore.ErrKeyNotFound {
				log.Debugf("failed to get networks for scope %s: %v", store.Scope(), err)
			}
			continue
		}

		for _, kvo := range kvol {
			n := kvo.(*network)
			n.Lock()
			n.ctrlr = c
			n.Unlock()

			ec := &endpointCnt{n: n}
			err = store.GetObject(datastore.Key(ec.Key()...), ec)
			if err != nil && !n.inDelete {
				log.Warnf("could not find endpoint count key %s for network %s while listing: %v", datastore.Key(ec.Key()...), n.Name(), err)
				continue
			}

			n.Lock()
			n.epCnt = ec
			n.scope = store.Scope()
			n.Unlock()
			nl = append(nl, n)
		}
	}

	return nl, nil
}

func (n *network) getEndpointFromStore(eid string) (*endpoint, error) {
	var errors []string
	for _, store := range n.ctrlr.getStores() {
		ep := &endpoint{id: eid, network: n}
		err := store.GetObject(datastore.Key(ep.Key()...), ep)
		// Continue searching in the next store if the key is not found in this store
		if err != nil {
			if err != datastore.ErrKeyNotFound {
				errors = append(errors, fmt.Sprintf("{%s:%v}, ", store.Scope(), err))
				log.Debugf("could not find endpoint %s in %s: %v", eid, store.Scope(), err)
			}
			continue
		}
		return ep, nil
	}
	return nil, fmt.Errorf("could not find endpoint %s: %v", eid, errors)
}

func (n *network) getEndpointsFromStore() ([]*endpoint, error) {
	var epl []*endpoint

	tmp := endpoint{network: n}
	for _, store := range n.getController().getStores() {
		kvol, err := store.List(datastore.Key(tmp.KeyPrefix()...), &endpoint{network: n})
		// Continue searching in the next store if no keys found in this store
		if err != nil {
			if err != datastore.ErrKeyNotFound {
				log.Debugf("failed to get endpoints for network %s scope %s: %v",
					n.Name(), store.Scope(), err)
			}
			continue
		}

		for _, kvo := range kvol {
			ep := kvo.(*endpoint)
			epl = append(epl, ep)
		}
	}

	return epl, nil
}

func (c *controller) updateToStore(kvObject datastore.KVObject) error {
	cs := c.getStore(kvObject.DataScope())
	if cs == nil {
		return fmt.Errorf("datastore for scope %q is not initialized ", kvObject.DataScope())
	}

	if err := cs.PutObjectAtomic(kvObject); err != nil {
		if err == datastore.ErrKeyModified {
			return err
		}
		return fmt.Errorf("failed to update store for object type %T: %v", kvObject, err)
	}

	return nil
}

func (c *controller) deleteFromStore(kvObject datastore.KVObject) error {
	cs := c.getStore(kvObject.DataScope())
	if cs == nil {
		return fmt.Errorf("datastore for scope %q is not initialized ", kvObject.DataScope())
	}

retry:
	if err := cs.DeleteObjectAtomic(kvObject); err != nil {
		if err == datastore.ErrKeyModified {
			if err := cs.GetObject(datastore.Key(kvObject.Key()...), kvObject); err != nil {
				return fmt.Errorf("could not update the kvobject to latest when trying to delete: %v", err)
			}
			goto retry
		}
		return err
	}

	return nil
}

type netWatch struct {
	localEps  map[string]*endpoint
	remoteEps map[string]*endpoint
	stopCh    chan struct{}
}

func (c *controller) getLocalEps(nw *netWatch) []*endpoint {
	c.Lock()
	defer c.Unlock()

	var epl []*endpoint
	for _, ep := range nw.localEps {
		epl = append(epl, ep)
	}

	return epl
}

func (c *controller) watchSvcRecord(ep *endpoint) {
	c.watchCh <- ep
}

func (c *controller) unWatchSvcRecord(ep *endpoint) {
	c.unWatchCh <- ep
}

func (c *controller) networkWatchLoop(nw *netWatch, ep *endpoint, ecCh <-chan datastore.KVObject) {
	for {
		select {
		case <-nw.stopCh:
			return
		case o := <-ecCh:
			ec := o.(*endpointCnt)

			epl, err := ec.n.getEndpointsFromStore()
			if err != nil {
				break
			}

			c.Lock()
			var addEp []*endpoint

			delEpMap := make(map[string]*endpoint)
			renameEpMap := make(map[string]bool)
			for k, v := range nw.remoteEps {
				delEpMap[k] = v
			}

			for _, lEp := range epl {
				if _, ok := nw.localEps[lEp.ID()]; ok {
					continue
				}

				if ep, ok := nw.remoteEps[lEp.ID()]; ok {
					// On a container rename EP ID will remain
					// the same but the name will change. service
					// records should reflect the change.
					// Keep old EP entry in the delEpMap and add
					// EP from the store (which has the new name)
					// into the new list
					if lEp.name == ep.name {
						delete(delEpMap, lEp.ID())
						continue
					}
					renameEpMap[lEp.ID()] = true
				}
				nw.remoteEps[lEp.ID()] = lEp
				addEp = append(addEp, lEp)
			}

			// EPs whose name are to be deleted from the svc records
			// should also be removed from nw's remote EP list, except
			// the ones that are getting renamed.
			for _, lEp := range delEpMap {
				if !renameEpMap[lEp.ID()] {
					delete(nw.remoteEps, lEp.ID())
				}
			}
			c.Unlock()

			for _, lEp := range delEpMap {
				ep.getNetwork().updateSvcRecord(lEp, c.getLocalEps(nw), false)

			}
			for _, lEp := range addEp {
				ep.getNetwork().updateSvcRecord(lEp, c.getLocalEps(nw), true)
			}
		}
	}
}

func (c *controller) processEndpointCreate(nmap map[string]*netWatch, ep *endpoint) {
	if !c.isDistributedControl() && ep.getNetwork().driverScope() == datastore.GlobalScope {
		return
	}

	c.Lock()
	nw, ok := nmap[ep.getNetwork().ID()]
	c.Unlock()

	if ok {
		// Update the svc db for the local endpoint join right away
		ep.getNetwork().updateSvcRecord(ep, c.getLocalEps(nw), true)

		c.Lock()
		nw.localEps[ep.ID()] = ep

		// If we had learned that from the kv store remove it
		// from remote ep list now that we know that this is
		// indeed a local endpoint
		delete(nw.remoteEps, ep.ID())
		c.Unlock()
		return
	}

	nw = &netWatch{
		localEps:  make(map[string]*endpoint),
		remoteEps: make(map[string]*endpoint),
	}

	// Update the svc db for the local endpoint join right away
	// Do this before adding this ep to localEps so that we don't
	// try to update this ep's container's svc records
	ep.getNetwork().updateSvcRecord(ep, c.getLocalEps(nw), true)

	c.Lock()
	nw.localEps[ep.ID()] = ep
	nmap[ep.getNetwork().ID()] = nw
	nw.stopCh = make(chan struct{})
	c.Unlock()

	store := c.getStore(ep.getNetwork().DataScope())
	if store == nil {
		return
	}

	if !store.Watchable() {
		return
	}

	ch, err := store.Watch(ep.getNetwork().getEpCnt(), nw.stopCh)
	if err != nil {
		log.Warnf("Error creating watch for network: %v", err)
		return
	}

	go c.networkWatchLoop(nw, ep, ch)
}

func (c *controller) processEndpointDelete(nmap map[string]*netWatch, ep *endpoint) {
	if !c.isDistributedControl() && ep.getNetwork().driverScope() == datastore.GlobalScope {
		return
	}

	c.Lock()
	nw, ok := nmap[ep.getNetwork().ID()]

	if ok {
		delete(nw.localEps, ep.ID())
		c.Unlock()

		// Update the svc db about local endpoint leave right away
		// Do this after we remove this ep from localEps so that we
		// don't try to remove this svc record from this ep's container.
		ep.getNetwork().updateSvcRecord(ep, c.getLocalEps(nw), false)

		c.Lock()
		if len(nw.localEps) == 0 {
			close(nw.stopCh)

			// This is the last container going away for the network. Destroy
			// this network's svc db entry
			delete(c.svcRecords, ep.getNetwork().ID())

			delete(nmap, ep.getNetwork().ID())
		}
	}
	c.Unlock()
}

func (c *controller) watchLoop() {
	for {
		select {
		case ep := <-c.watchCh:
			c.processEndpointCreate(c.nmap, ep)
		case ep := <-c.unWatchCh:
			c.processEndpointDelete(c.nmap, ep)
		}
	}
}

func (c *controller) startWatch() {
	if c.watchCh != nil {
		return
	}
	c.watchCh = make(chan *endpoint)
	c.unWatchCh = make(chan *endpoint)
	c.nmap = make(map[string]*netWatch)

	go c.watchLoop()
}

func (c *controller) networkCleanup() {
	networks, err := c.getNetworksFromStore()
	if err != nil {
		log.Warnf("Could not retrieve networks from store(s) during network cleanup: %v", err)
		return
	}

	for _, n := range networks {
		if n.inDelete {
			log.Infof("Removing stale network %s (%s)", n.Name(), n.ID())
			if err := n.delete(true); err != nil {
				log.Debugf("Error while removing stale network: %v", err)
			}
		}
	}
}

var populateSpecial NetworkWalker = func(nw Network) bool {
	if n := nw.(*network); n.hasSpecialDriver() {
		if err := n.getController().addNetwork(n); err != nil {
			log.Warnf("Failed to populate network %q with driver %q", nw.Name(), nw.Type())
		}
	}
	return false
}
<|endoftext|>


<commit_before>package argot

import (
	"testing"
)

// Step represents a step in a test.
type Step interface {
	Go() error
}

// If we can have a single Step, then we can have a slice of Steps
// representing the order of Steps in a larger unit.
type Steps []Step

// Test runs the steps in order and returns either all the steps, or
// all the steps that did not error, plus the step that errored. Thus
// the results are always a prefix of the Steps.  t can be nil. If t
// is not nil and an error occurs, then t.Fatal will be called. If an
// error occurs, it will be returned.
func (ss Steps) Test(t *testing.T) (results Steps, err error) {
	if t != nil {
		defer func() {
			if err != nil {
				t.Log("Achieved Steps:")
				for _, result := range results {
					t.Logf("  %v", result)
				}
				t.Fatalf("Error: %v", err)
			}
		}()
	}
	for idx, step := range ss {
		err = step.Go()
		if err != nil {
			return ss[:idx], err
		}
	}
	return ss, nil
}

// Steps are useful, but there are times where you don't know ahead of
// time exactly which Steps you wish to run. Consider a test where the
// steps are dependent on the response you receive from a server.
// Thus the advantage of using a chan is that it allows more laziness:
// steps can be responsible for issuing their own subsequent steps.
type StepsChan <-chan Step

// Test runs the steps in order and returns either all the steps, or
// all the steps that did not error, plus the step that errored. Thus
// the results are always a prefix of the Steps.  t can be nil. If t
// is not nil and an error occurs, then t.Fatal will be called. If an
// error occurs, it will be returned.
func (sc StepsChan) Test(t *testing.T) (results Steps, err error) {
	if t != nil {
		defer func() {
			if err != nil {
				t.Log("Achieved Steps:")
				for _, result := range results {
					t.Logf("  %v", result)
				}
				t.Fatalf("Error: %v", err)
			}
		}()
	}
	results = make([]Step, 0, 16)
	for step := range sc {
		results = append(results, step)
		err = step.Go()
		if err != nil {
			return results, err
		}
	}
	return results, nil
}

// StepFunc is the basic type of a Step: a function that takes no
// arguments and returns an error.
type StepFunc func() error

func (sf StepFunc) Go() error {
	return sf()
}

// NamedStep extends StepFunc by adding a name, which is mainly of use
// when formatting a Step.
type NamedStep struct {
	StepFunc
	name string
}

func (ns NamedStep) String() string {
	return ns.name
}

// NewNamedStep creates a NamedStep with the given name and Step
// function.
func NewNamedStep(name string, step StepFunc) *NamedStep {
	return &NamedStep{
		StepFunc: step,
		name:     name,
	}
}

// AnyError is a utility function that returns the first non-nil error
// in the slice, or nil if either the slice or all elements of the
// slice are nil.
func AnyError(errs ...error) error {
	for _, err := range errs {
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Format steps via pretty printer.<commit_after>package argot

import (
	"fmt"
	"reflect"
	"testing"

	"github.com/kylelemons/godebug/pretty"
)

// Step represents a step in a test.
type Step interface {
	Go() error
}

// If we can have a single Step, then we can have a slice of Steps
// representing the order of Steps in a larger unit.
type Steps []Step

var (
	defaultConfig = &pretty.Config{
		Formatter: map[reflect.Type]interface{}{
			reflect.TypeOf((*Step)(nil)).Elem(): fmt.Sprint,
		},
	}
)

func formatFatalSteps(results Steps, err error) string {
	msg := ""
	if len(results) > 0 {
		msg = "Achieved Steps:\n" + defaultConfig.Sprint(results) + "\n"
	}
	return fmt.Sprintf("%vError: %v", msg, err)
}

// Test runs the steps in order and returns either all the steps, or
// all the steps that did not error, plus the step that errored. Thus
// the results are always a prefix of the Steps.  t can be nil. If t
// is not nil and an error occurs, then t.Fatal will be called. If an
// error occurs, it will be returned.
func (ss Steps) Test(t *testing.T) (results Steps, err error) {
	if t != nil {
		defer func() {
			if err != nil {
				t.Fatalf(formatFatalSteps(results, err))
			}
		}()
	}
	for idx, step := range ss {
		err = step.Go()
		if err != nil {
			return ss[:idx], err
		}
	}
	return ss, nil
}

// Steps are useful, but there are times where you don't know ahead of
// time exactly which Steps you wish to run. Consider a test where the
// steps are dependent on the response you receive from a server.
// Thus the advantage of using a chan is that it allows more laziness:
// steps can be responsible for issuing their own subsequent steps.
type StepsChan <-chan Step

// Test runs the steps in order and returns either all the steps, or
// all the steps that did not error, plus the step that errored. Thus
// the results are always a prefix of the Steps.  t can be nil. If t
// is not nil and an error occurs, then t.Fatal will be called. If an
// error occurs, it will be returned.
func (sc StepsChan) Test(t *testing.T) (results Steps, err error) {
	if t != nil {
		defer func() {
			if err != nil {
				t.Fatalf(formatFatalSteps(results, err))
			}
		}()
	}
	results = make([]Step, 0, 16)
	for step := range sc {
		results = append(results, step)
		err = step.Go()
		if err != nil {
			return results, err
		}
	}
	return results, nil
}

// StepFunc is the basic type of a Step: a function that takes no
// arguments and returns an error.
type StepFunc func() error

func (sf StepFunc) Go() error {
	return sf()
}

// NamedStep extends StepFunc by adding a name, which is mainly of use
// when formatting a Step.
type NamedStep struct {
	StepFunc
	name string
}

func (ns NamedStep) String() string {
	return ns.name
}

// NewNamedStep creates a NamedStep with the given name and Step
// function.
func NewNamedStep(name string, step StepFunc) *NamedStep {
	return &NamedStep{
		StepFunc: step,
		name:     name,
	}
}

// AnyError is a utility function that returns the first non-nil error
// in the slice, or nil if either the slice or all elements of the
// slice are nil.
func AnyError(errs ...error) error {
	for _, err := range errs {
		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package papernet

import (
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var arxivSummaryPipe = CleaningPipe(
	strings.TrimSpace,
	OneLine,
	strings.TrimSpace,
)

var arxivCategories = map[string]string{
	"stat.AP":            "Statistics - Applications",
	"stat.CO":            "Statistics - Computation",
	"stat.ML":            "Statistics - Machine Learning",
	"stat.ME":            "Statistics - Methodology",
	"stat.TH":            "Statistics - Theory",
	"q-bio.BM":           "Quantitative Biology - Biomolecules",
	"q-bio.CB":           "Quantitative Biology - Cell Behavior",
	"q-bio.GN":           "Quantitative Biology - Genomics",
	"q-bio.MN":           "Quantitative Biology - Molecular Networks",
	"q-bio.NC":           "Quantitative Biology - Neurons and Cognition",
	"q-bio.OT":           "Quantitative Biology - Other",
	"q-bio.PE":           "Quantitative Biology - Populations and Evolution",
	"q-bio.QM":           "Quantitative Biology - Quantitative Methods",
	"q-bio.SC":           "Quantitative Biology - Subcellular Processes",
	"q-bio.TO":           "Quantitative Biology - Tissues and Organs",
	"cs.AR":              "Computer Science - Architecture",
	"cs.AI":              "Computer Science - Artificial Intelligence",
	"cs.CL":              "Computer Science - Computation and Language",
	"cs.CC":              "Computer Science - Computational Complexity",
	"cs.CE":              "Computer Science - Computational Engineering; Finance; and Science",
	"cs.CG":              "Computer Science - Computational Geometry",
	"cs.GT":              "Computer Science - Computer Science and Game Theory",
	"cs.CV":              "Computer Science - Computer Vision and Pattern Recognition",
	"cs.CY":              "Computer Science - Computers and Society",
	"cs.CR":              "Computer Science - Cryptography and Security",
	"cs.DS":              "Computer Science - Data Structures and Algorithms",
	"cs.DB":              "Computer Science - Databases",
	"cs.DL":              "Computer Science - Digital Libraries",
	"cs.DM":              "Computer Science - Discrete Mathematics",
	"cs.DC":              "Computer Science - Distributed; Parallel; and Cluster Computing",
	"cs.GL":              "Computer Science - General Literature",
	"cs.GR":              "Computer Science - Graphics",
	"cs.HC":              "Computer Science - Human-Computer Interaction",
	"cs.IR":              "Computer Science - Information Retrieval",
	"cs.IT":              "Computer Science - Information Theory",
	"cs.LG":              "Computer Science - Learning",
	"cs.LO":              "Computer Science - Logic in Computer Science",
	"cs.MS":              "Computer Science - Mathematical Software",
	"cs.MA":              "Computer Science - Multiagent Systems",
	"cs.MM":              "Computer Science - Multimedia",
	"cs.NI":              "Computer Science - Networking and Internet Architecture",
	"cs.NE":              "Computer Science - Neural and Evolutionary Computing",
	"cs.NA":              "Computer Science - Numerical Analysis",
	"cs.OS":              "Computer Science - Operating Systems",
	"cs.OH":              "Computer Science - Other",
	"cs.PF":              "Computer Science - Performance",
	"cs.PL":              "Computer Science - Programming Languages",
	"cs.RO":              "Computer Science - Robotics",
	"cs.SE":              "Computer Science - Software Engineering",
	"cs.SD":              "Computer Science - Sound",
	"cs.SC":              "Computer Science - Symbolic Computation",
	"nlin.AO":            "Nonlinear Sciences - Adaptation and Self-Organizing Systems",
	"nlin.CG":            "Nonlinear Sciences - Cellular Automata and Lattice Gases",
	"nlin.CD":            "Nonlinear Sciences - Chaotic Dynamics",
	"nlin.SI":            "Nonlinear Sciences - Exactly Solvable and Integrable Systems",
	"nlin.PS":            "Nonlinear Sciences - Pattern Formation and Solitons",
	"math.AG":            "Mathematics - Algebraic Geometry",
	"math.AT":            "Mathematics - Algebraic Topology",
	"math.AP":            "Mathematics - Analysis of PDEs",
	"math.CT":            "Mathematics - Category Theory",
	"math.CA":            "Mathematics - Classical Analysis and ODEs",
	"math.CO":            "Mathematics - Combinatorics",
	"math.AC":            "Mathematics - Commutative Algebra",
	"math.CV":            "Mathematics - Complex Variables",
	"math.DG":            "Mathematics - Differential Geometry",
	"math.DS":            "Mathematics - Dynamical Systems",
	"math.FA":            "Mathematics - Functional Analysis",
	"math.GM":            "Mathematics - General Mathematics",
	"math.GN":            "Mathematics - General Topology",
	"math.GT":            "Mathematics - Geometric Topology",
	"math.GR":            "Mathematics - Group Theory",
	"math.HO":            "Mathematics - History and Overview",
	"math.IT":            "Mathematics - Information Theory",
	"math.KT":            "Mathematics - K-Theory and Homology",
	"math.LO":            "Mathematics - Logic",
	"math.MP":            "Mathematics - Mathematical Physics",
	"math.MG":            "Mathematics - Metric Geometry",
	"math.NT":            "Mathematics - Number Theory",
	"math.NA":            "Mathematics - Numerical Analysis",
	"math.OA":            "Mathematics - Operator Algebras",
	"math.OC":            "Mathematics - Optimization and Control",
	"math.PR":            "Mathematics - Probability",
	"math.QA":            "Mathematics - Quantum Algebra",
	"math.RT":            "Mathematics - Representation Theory",
	"math.RA":            "Mathematics - Rings and Algebras",
	"math.SP":            "Mathematics - Spectral Theory",
	"math.ST":            "Mathematics - Statistics",
	"math.SG":            "Mathematics - Symplectic Geometry",
	"astro-ph":           "Astrophysics",
	"cond-mat.dis-nn":    "Physics - Disordered Systems and Neural Networks",
	"cond-mat.mes-hall":  "Physics - Mesoscopic Systems and Quantum Hall Effect",
	"cond-mat.mtrl-sci":  "Physics - Materials Science",
	"cond-mat.other":     "Physics - Other",
	"cond-mat.soft":      "Physics - Soft Condensed Matter",
	"cond-mat.stat-mech": "Physics - Statistical Mechanics",
	"cond-mat.str-el":    "Physics - Strongly Correlated Electrons",
	"cond-mat.supr-con":  "Physics - Superconductivity",
	"gr-qc":              "General Relativity and Quantum Cosmology",
	"hep-ex":             "High Energy Physics - Experiment",
	"hep-lat":            "High Energy Physics - Lattice",
	"hep-ph":             "High Energy Physics - Phenomenology",
	"hep-th":             "High Energy Physics - Theory",
	"math-ph":            "Mathematical Physics",
	"nucl-ex":            "Nuclear Experiment",
	"nucl-th":            "Nuclear Theory",
	"physics.acc-ph":     "Physics - Accelerator Physics",
	"physics.ao-ph":      "Physics - Atmospheric and Oceanic Physics",
	"physics.atom-ph":    "Physics - Atomic Physics",
	"physics.atm-clus":   "Physics - Atomic and Molecular Clusters",
	"physics.bio-ph":     "Physics - Biological Physics",
	"physics.chem-ph":    "Physics - Chemical Physics",
	"physics.class-ph":   "Physics - Classical Physics",
	"physics.comp-ph":    "Physics - Computational Physics",
	"physics.data-an":    "Physics - Data Analysis; Statistics and Probability",
	"physics.flu-dyn":    "Physics - Fluid Dynamics",
	"physics.gen-ph":     "Physics - General Physics",
	"physics.geo-ph":     "Physics - Geophysics",
	"physics.hist-ph":    "Physics - History of Physics",
	"physics.ins-det":    "Physics - Instrumentation and Detectors",
	"physics.med-ph":     "Physics - Medical Physics",
	"physics.optics":     "Physics - Optics",
	"physics.ed-ph":      "Physics - Physics Education",
	"physics.soc-ph":     "Physics - Physics and Society",
	"physics.plasm-ph":   "Physics - Plasma Physics",
	"physics.pop-ph":     "Physics - Popular Physics",
	"physics.space-ph":   "Physics - Space Physics",
	"quant-ph":           "Quantum Physics",
}

var arxivRegExp *regexp.Regexp

func init() {
	arxivRegExp, _ = regexp.Compile("http://arxiv.org/abs/([0-9.]*)(v[0-9]+)?")
}

type ArxivSearch struct {
	Q          string
	Start      int
	MaxResults int
}

type ArxivResult struct {
	Papers     []*Paper
	Pagination Pagination
}

type ArxivSpider struct {
	Client *http.Client
}

func (s *ArxivSpider) Search(search ArxivSearch) (ArxivResult, error) {
	u, _ := url.Parse("http://export.arxiv.org/api/query")

	query := u.Query()

	if search.Q != "" {
		query.Add("search_query", fmt.Sprintf("all:%s", search.Q))
	}
	if search.Start > 0 {
		query.Add("start", strconv.Itoa(search.Start))
	}
	if search.MaxResults > 0 {
		query.Add("max_results", strconv.Itoa(search.MaxResults))
	}

	query.Add("sortBy", "submittedDate")
	query.Add("sortOrder", "descending")

	u.RawQuery = query.Encode()

	resp, err := s.Client.Get(u.String())
	if err != nil {
		return ArxivResult{}, err
	}

	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return ArxivResult{}, err
	}

	r := struct {
		Title string `xml:"title"`
		ID    string `xml:"id"`
		Total struct {
			Value uint64 `xml:",chardata"`
		} `xml:"totalResults"`
		Offset struct {
			Value uint64 `xml:",chardata"`
		} `xml:"startIndex"`
		Limit struct {
			Value uint64 `xml:",chardata"`
		} `xml:"itemsPerPage"`
		Entries []struct {
			Title   string `xml:"title"`
			ID      string `xml:"id"`
			Summary string `xml:"summary"`
			Links   []struct {
				HRef string `xml:"href,attr"`
				Type string `xml:"type,attr"`
			} `xml:"link"`
			Categories []struct {
				Term string `xml:"term,attr"`
			} `xml:"category"`
			Published time.Time `xml:"published"`
			Updated   time.Time `xml:"updated"`
		} `xml:"entry"`
	}{}
	err = xml.Unmarshal(data, &r)
	if err != nil {
		return ArxivResult{}, err
	}

	papers := make([]*Paper, len(r.Entries))
	for i, entry := range r.Entries {
		tags := make([]string, 0, len(entry.Categories))
		for _, cat := range entry.Categories {
			tag, ok := arxivCategories[cat.Term]
			if ok {
				tags = append(tags, tag)
			}
		}

		var arxivID string
		matches := arxivRegExp.FindAllStringSubmatch(entry.ID, -1)
		if len(matches) > 0 && len(matches[0]) > 1 {
			arxivID = matches[0][1]
		}

		papers[i] = &Paper{
			Title:   entry.Title,
			Summary: arxivSummaryPipe(entry.Summary),
			References: []string{
				entry.Links[0].HRef, // link to arXiv
				entry.Links[1].HRef, // PDF
			},
			Tags:      tags,
			CreatedAt: entry.Published,
			UpdatedAt: entry.Updated,
			ArxivID:   arxivID,
		}
	}

	return ArxivResult{
		Papers: papers,
		Pagination: Pagination{
			Total:  r.Total.Value,
			Limit:  r.Limit.Value,
			Offset: r.Offset.Value,
		},
	}, nil
}
<commit_msg>Search arXiv with ANDs<commit_after>package papernet

import (
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var arxivSummaryPipe = CleaningPipe(
	strings.TrimSpace,
	OneLine,
	strings.TrimSpace,
)

var arxivCategories = map[string]string{
	"stat.AP":            "Statistics - Applications",
	"stat.CO":            "Statistics - Computation",
	"stat.ML":            "Statistics - Machine Learning",
	"stat.ME":            "Statistics - Methodology",
	"stat.TH":            "Statistics - Theory",
	"q-bio.BM":           "Quantitative Biology - Biomolecules",
	"q-bio.CB":           "Quantitative Biology - Cell Behavior",
	"q-bio.GN":           "Quantitative Biology - Genomics",
	"q-bio.MN":           "Quantitative Biology - Molecular Networks",
	"q-bio.NC":           "Quantitative Biology - Neurons and Cognition",
	"q-bio.OT":           "Quantitative Biology - Other",
	"q-bio.PE":           "Quantitative Biology - Populations and Evolution",
	"q-bio.QM":           "Quantitative Biology - Quantitative Methods",
	"q-bio.SC":           "Quantitative Biology - Subcellular Processes",
	"q-bio.TO":           "Quantitative Biology - Tissues and Organs",
	"cs.AR":              "Computer Science - Architecture",
	"cs.AI":              "Computer Science - Artificial Intelligence",
	"cs.CL":              "Computer Science - Computation and Language",
	"cs.CC":              "Computer Science - Computational Complexity",
	"cs.CE":              "Computer Science - Computational Engineering; Finance; and Science",
	"cs.CG":              "Computer Science - Computational Geometry",
	"cs.GT":              "Computer Science - Computer Science and Game Theory",
	"cs.CV":              "Computer Science - Computer Vision and Pattern Recognition",
	"cs.CY":              "Computer Science - Computers and Society",
	"cs.CR":              "Computer Science - Cryptography and Security",
	"cs.DS":              "Computer Science - Data Structures and Algorithms",
	"cs.DB":              "Computer Science - Databases",
	"cs.DL":              "Computer Science - Digital Libraries",
	"cs.DM":              "Computer Science - Discrete Mathematics",
	"cs.DC":              "Computer Science - Distributed; Parallel; and Cluster Computing",
	"cs.GL":              "Computer Science - General Literature",
	"cs.GR":              "Computer Science - Graphics",
	"cs.HC":              "Computer Science - Human-Computer Interaction",
	"cs.IR":              "Computer Science - Information Retrieval",
	"cs.IT":              "Computer Science - Information Theory",
	"cs.LG":              "Computer Science - Learning",
	"cs.LO":              "Computer Science - Logic in Computer Science",
	"cs.MS":              "Computer Science - Mathematical Software",
	"cs.MA":              "Computer Science - Multiagent Systems",
	"cs.MM":              "Computer Science - Multimedia",
	"cs.NI":              "Computer Science - Networking and Internet Architecture",
	"cs.NE":              "Computer Science - Neural and Evolutionary Computing",
	"cs.NA":              "Computer Science - Numerical Analysis",
	"cs.OS":              "Computer Science - Operating Systems",
	"cs.OH":              "Computer Science - Other",
	"cs.PF":              "Computer Science - Performance",
	"cs.PL":              "Computer Science - Programming Languages",
	"cs.RO":              "Computer Science - Robotics",
	"cs.SE":              "Computer Science - Software Engineering",
	"cs.SD":              "Computer Science - Sound",
	"cs.SC":              "Computer Science - Symbolic Computation",
	"nlin.AO":            "Nonlinear Sciences - Adaptation and Self-Organizing Systems",
	"nlin.CG":            "Nonlinear Sciences - Cellular Automata and Lattice Gases",
	"nlin.CD":            "Nonlinear Sciences - Chaotic Dynamics",
	"nlin.SI":            "Nonlinear Sciences - Exactly Solvable and Integrable Systems",
	"nlin.PS":            "Nonlinear Sciences - Pattern Formation and Solitons",
	"math.AG":            "Mathematics - Algebraic Geometry",
	"math.AT":            "Mathematics - Algebraic Topology",
	"math.AP":            "Mathematics - Analysis of PDEs",
	"math.CT":            "Mathematics - Category Theory",
	"math.CA":            "Mathematics - Classical Analysis and ODEs",
	"math.CO":            "Mathematics - Combinatorics",
	"math.AC":            "Mathematics - Commutative Algebra",
	"math.CV":            "Mathematics - Complex Variables",
	"math.DG":            "Mathematics - Differential Geometry",
	"math.DS":            "Mathematics - Dynamical Systems",
	"math.FA":            "Mathematics - Functional Analysis",
	"math.GM":            "Mathematics - General Mathematics",
	"math.GN":            "Mathematics - General Topology",
	"math.GT":            "Mathematics - Geometric Topology",
	"math.GR":            "Mathematics - Group Theory",
	"math.HO":            "Mathematics - History and Overview",
	"math.IT":            "Mathematics - Information Theory",
	"math.KT":            "Mathematics - K-Theory and Homology",
	"math.LO":            "Mathematics - Logic",
	"math.MP":            "Mathematics - Mathematical Physics",
	"math.MG":            "Mathematics - Metric Geometry",
	"math.NT":            "Mathematics - Number Theory",
	"math.NA":            "Mathematics - Numerical Analysis",
	"math.OA":            "Mathematics - Operator Algebras",
	"math.OC":            "Mathematics - Optimization and Control",
	"math.PR":            "Mathematics - Probability",
	"math.QA":            "Mathematics - Quantum Algebra",
	"math.RT":            "Mathematics - Representation Theory",
	"math.RA":            "Mathematics - Rings and Algebras",
	"math.SP":            "Mathematics - Spectral Theory",
	"math.ST":            "Mathematics - Statistics",
	"math.SG":            "Mathematics - Symplectic Geometry",
	"astro-ph":           "Astrophysics",
	"cond-mat.dis-nn":    "Physics - Disordered Systems and Neural Networks",
	"cond-mat.mes-hall":  "Physics - Mesoscopic Systems and Quantum Hall Effect",
	"cond-mat.mtrl-sci":  "Physics - Materials Science",
	"cond-mat.other":     "Physics - Other",
	"cond-mat.soft":      "Physics - Soft Condensed Matter",
	"cond-mat.stat-mech": "Physics - Statistical Mechanics",
	"cond-mat.str-el":    "Physics - Strongly Correlated Electrons",
	"cond-mat.supr-con":  "Physics - Superconductivity",
	"gr-qc":              "General Relativity and Quantum Cosmology",
	"hep-ex":             "High Energy Physics - Experiment",
	"hep-lat":            "High Energy Physics - Lattice",
	"hep-ph":             "High Energy Physics - Phenomenology",
	"hep-th":             "High Energy Physics - Theory",
	"math-ph":            "Mathematical Physics",
	"nucl-ex":            "Nuclear Experiment",
	"nucl-th":            "Nuclear Theory",
	"physics.acc-ph":     "Physics - Accelerator Physics",
	"physics.ao-ph":      "Physics - Atmospheric and Oceanic Physics",
	"physics.atom-ph":    "Physics - Atomic Physics",
	"physics.atm-clus":   "Physics - Atomic and Molecular Clusters",
	"physics.bio-ph":     "Physics - Biological Physics",
	"physics.chem-ph":    "Physics - Chemical Physics",
	"physics.class-ph":   "Physics - Classical Physics",
	"physics.comp-ph":    "Physics - Computational Physics",
	"physics.data-an":    "Physics - Data Analysis; Statistics and Probability",
	"physics.flu-dyn":    "Physics - Fluid Dynamics",
	"physics.gen-ph":     "Physics - General Physics",
	"physics.geo-ph":     "Physics - Geophysics",
	"physics.hist-ph":    "Physics - History of Physics",
	"physics.ins-det":    "Physics - Instrumentation and Detectors",
	"physics.med-ph":     "Physics - Medical Physics",
	"physics.optics":     "Physics - Optics",
	"physics.ed-ph":      "Physics - Physics Education",
	"physics.soc-ph":     "Physics - Physics and Society",
	"physics.plasm-ph":   "Physics - Plasma Physics",
	"physics.pop-ph":     "Physics - Popular Physics",
	"physics.space-ph":   "Physics - Space Physics",
	"quant-ph":           "Quantum Physics",
}

var arxivRegExp *regexp.Regexp

func init() {
	arxivRegExp, _ = regexp.Compile("http://arxiv.org/abs/([0-9.]*)(v[0-9]+)?")
}

type ArxivSearch struct {
	Q          string
	Start      int
	MaxResults int
}

type ArxivResult struct {
	Papers     []*Paper
	Pagination Pagination
}

type ArxivSpider struct {
	Client *http.Client
}

func (s *ArxivSpider) Search(search ArxivSearch) (ArxivResult, error) {
	u, _ := url.Parse("http://export.arxiv.org/api/query")

	query := u.Query()

	if search.Q != "" {
		re, _ := regexp.Compile("[A-Za-z0-9]+")
		matches := re.FindAllStringSubmatch(search.Q, -1)
		fmt.Println(matches)
		q := make([]string, len(matches))
		for i, match := range matches {
			q[i] = match[0]
		}
		query.Add("search_query", fmt.Sprintf("all:%s", strings.Join(q, " AND ")))
	}
	if search.Start > 0 {
		query.Add("start", strconv.Itoa(search.Start))
	}
	if search.MaxResults > 0 {
		query.Add("max_results", strconv.Itoa(search.MaxResults))
	}

	query.Add("sortBy", "submittedDate")
	query.Add("sortOrder", "descending")

	u.RawQuery = query.Encode()

	resp, err := s.Client.Get(u.String())
	if err != nil {
		return ArxivResult{}, err
	}

	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return ArxivResult{}, err
	}

	r := struct {
		Title string `xml:"title"`
		ID    string `xml:"id"`
		Total struct {
			Value uint64 `xml:",chardata"`
		} `xml:"totalResults"`
		Offset struct {
			Value uint64 `xml:",chardata"`
		} `xml:"startIndex"`
		Limit struct {
			Value uint64 `xml:",chardata"`
		} `xml:"itemsPerPage"`
		Entries []struct {
			Title   string `xml:"title"`
			ID      string `xml:"id"`
			Summary string `xml:"summary"`
			Links   []struct {
				HRef string `xml:"href,attr"`
				Type string `xml:"type,attr"`
			} `xml:"link"`
			Categories []struct {
				Term string `xml:"term,attr"`
			} `xml:"category"`
			Published time.Time `xml:"published"`
			Updated   time.Time `xml:"updated"`
		} `xml:"entry"`
	}{}
	err = xml.Unmarshal(data, &r)
	if err != nil {
		return ArxivResult{}, err
	}

	papers := make([]*Paper, len(r.Entries))
	for i, entry := range r.Entries {
		tags := make([]string, 0, len(entry.Categories))
		for _, cat := range entry.Categories {
			tag, ok := arxivCategories[cat.Term]
			if ok {
				tags = append(tags, tag)
			}
		}

		var arxivID string
		matches := arxivRegExp.FindAllStringSubmatch(entry.ID, -1)
		if len(matches) > 0 && len(matches[0]) > 1 {
			arxivID = matches[0][1]
		}

		papers[i] = &Paper{
			Title:   entry.Title,
			Summary: arxivSummaryPipe(entry.Summary),
			References: []string{
				entry.Links[0].HRef, // link to arXiv
				entry.Links[1].HRef, // PDF
			},
			Tags:      tags,
			CreatedAt: entry.Published,
			UpdatedAt: entry.Updated,
			ArxivID:   arxivID,
		}
	}

	return ArxivResult{
		Papers: papers,
		Pagination: Pagination{
			Total:  r.Total.Value,
			Limit:  r.Limit.Value,
			Offset: r.Offset.Value,
		},
	}, nil
}
<|endoftext|>


<commit_before>// +build linux

package linux

import (
	"bufio"
	"io"
	"os"
	"strings"

	"github.com/mackerelio/golib/logging"
)

// CPUGenerator Collects CPU specs
type CPUGenerator struct {
}

// Key return "cpu"
func (g *CPUGenerator) Key() string {
	return "cpu"
}

var cpuLogger = logging.GetLogger("spec.cpu")

func (g *CPUGenerator) generate(file io.Reader) (interface{}, error) {
	scanner := bufio.NewScanner(file)

	var results []map[string]interface{}
	var cur map[string]interface{}
	var modelName string

	for scanner.Scan() {
		line := scanner.Text()
		kv := strings.SplitN(line, ":", 2)
		if len(kv) < 2 {
			continue
		}
		key := strings.TrimSpace(kv[0])
		val := strings.TrimSpace(kv[1])
		switch key {
		case "processor":
			cur = make(map[string]interface{})
			if modelName != "" {
				cur["model_name"] = modelName
			}
			results = append(results, cur)
		case "Processor":
			modelName = val
		case "vendor_id", "model", "stepping", "physical id", "core id", "model name", "cache size":
			cur[strings.Replace(key, " ", "_", -1)] = val
		case "cpu family":
			cur["family"] = val
		case "cpu cores":
			cur["cores"] = val
		case "cpu MHz":
			cur["mhz"] = val
		}
	}
	if err := scanner.Err(); err != nil {
		cpuLogger.Errorf("Failed (skip this spec): %s", err)
		return nil, err
	}

	// Old kernels with CONFIG_SMP disabled has no "processor: " line
	if len(results) == 0 && modelName != "" {
		cur = make(map[string]interface{})
		cur["model_name"] = modelName
		results = append(results, cur)
	}

	return results, nil
}

// Generate cpu specs
func (g *CPUGenerator) Generate() (interface{}, error) {
	file, err := os.Open("/proc/cpuinfo")
	if err != nil {
		cpuLogger.Errorf("Failed (skip this spec): %s", err)
		return nil, err
	}
	defer file.Close()

	return g.generate(file)
}
<commit_msg>Show CPU/SoC model name on Linux/MIPS<commit_after>// +build linux

package linux

import (
	"bufio"
	"io"
	"os"
	"strings"

	"github.com/mackerelio/golib/logging"
)

// CPUGenerator Collects CPU specs
type CPUGenerator struct {
}

// Key return "cpu"
func (g *CPUGenerator) Key() string {
	return "cpu"
}

var cpuLogger = logging.GetLogger("spec.cpu")

func (g *CPUGenerator) generate(file io.Reader) (interface{}, error) {
	scanner := bufio.NewScanner(file)

	var results []map[string]interface{}
	var cur map[string]interface{}
	var modelName string

	for scanner.Scan() {
		line := scanner.Text()
		kv := strings.SplitN(line, ":", 2)
		if len(kv) < 2 {
			continue
		}
		key := strings.TrimSpace(kv[0])
		val := strings.TrimSpace(kv[1])
		switch key {
		case "processor":
			cur = make(map[string]interface{})
			if modelName != "" {
				cur["model_name"] = modelName
			}
			results = append(results, cur)
		case "Processor", "system type":
			modelName = val
		case "vendor_id", "model", "stepping", "physical id", "core id", "model name", "cache size":
			cur[strings.Replace(key, " ", "_", -1)] = val
		case "cpu family":
			cur["family"] = val
		case "cpu cores":
			cur["cores"] = val
		case "cpu MHz":
			cur["mhz"] = val
		}
	}
	if err := scanner.Err(); err != nil {
		cpuLogger.Errorf("Failed (skip this spec): %s", err)
		return nil, err
	}

	// Old kernels with CONFIG_SMP disabled has no "processor: " line
	if len(results) == 0 && modelName != "" {
		cur = make(map[string]interface{})
		cur["model_name"] = modelName
		results = append(results, cur)
	}

	return results, nil
}

// Generate cpu specs
func (g *CPUGenerator) Generate() (interface{}, error) {
	file, err := os.Open("/proc/cpuinfo")
	if err != nil {
		cpuLogger.Errorf("Failed (skip this spec): %s", err)
		return nil, err
	}
	defer file.Close()

	return g.generate(file)
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/codegangsta/cli"
	sdk "github.com/dysolution/espsdk"
)

// The TermService uses the SDK to deserialize responses from endpoints that
// provide TermLists.
type TermService struct{ context *cli.Context }

// Unmarshal attempts to deserialize the provided JSON payload into a SubmissionBatch object.
func (m TermService) Unmarshal(payload []byte) sdk.TermList {
	return sdk.TermList{}.Unmarshal(payload)
}

// GetNumberOfPeople lists all possible values for Number of People.
func (m TermService) GetNumberOfPeople(context *cli.Context) sdk.TermList {
	return sdk.TermList{}.GetNumberOfPeople(&client)
}

// GetExpressions lists all possible facial expression values.
func (m TermService) GetExpressions(context *cli.Context) sdk.TermList {
	return sdk.TermList{}.GetExpressions(&client)
}

// GetCompositions lists all possible composition values.
func (m TermService) GetCompositions(context *cli.Context) sdk.TermList {
	return sdk.TermList{}.GetCompositions(&client)
}
<commit_msg>use sdk.Client{}.GetTermList instead of sdk.TermList{} methods<commit_after>package main

import (
	"github.com/codegangsta/cli"
	sdk "github.com/dysolution/espsdk"
)

// The TermService uses the SDK to deserialize responses from endpoints that
// provide TermLists.
type TermService struct{ context *cli.Context }

// Unmarshal attempts to deserialize the provided JSON payload into a SubmissionBatch object.
func (m TermService) Unmarshal(payload []byte) sdk.TermList {
	return sdk.TermList{}.Unmarshal(payload)
}

// GetNumberOfPeople lists all possible values for Number of People.
func (m TermService) GetNumberOfPeople(context *cli.Context) sdk.TermList {
	return client.GetTermList(sdk.NumberOfPeople)
}

// GetExpressions lists all possible facial expression values.
func (m TermService) GetExpressions(context *cli.Context) sdk.TermList {
	return client.GetTermList(sdk.Expressions)
}

// GetCompositions lists all possible composition values.
func (m TermService) GetCompositions(context *cli.Context) sdk.TermList {
	return client.GetTermList(sdk.Compositions)
}
<|endoftext|>


<commit_before>package kryptos

var (
	ENCRYPT_MAPPING = map[string]string{
		"a": "¡",
		"b": "¢",
		"c": "£",
		"d": "¤",
		"e": "¥",
		"f": "¦",
		"g": "§",
		"h": "¨",
		"i": "ª",
		"j": "«",
		"k": "¬",
		"l": "®",
		"m": "¯",
		"n": "°",
		"o": "©",
		"p": "±",
		"q": "²",
		"r": "³",
		"s": "´",
		"t": "µ",
		"u": "¶",
		"v": "·",
		"w": "¸",
		"x": "¹",
		"y": "º",
		"z": "»",
		"A": "¼",
		"B": "½",
		"C": "¾",
		"D": "¿",
		"E": "À",
		"F": "Á",
		"G": "Â",
		"H": "Ã",
		"I": "Ä",
		"J": "Å",
		"K": "Æ",
		"L": "Ç",
		"M": "È",
		"N": "É",
		"O": "Ê",
		"P": "Ë",
		"Q": "Ì",
		"R": "Í",
		"S": "Î",
		"T": "Ï",
		"U": "Ð",
		"V": "Ñ",
		"W": "Ò",
		"X": "Ó",
		"Y": "Ô",
		"Z": "Õ",
		"0": "Ö",
		"1": "×",
		"2": "Ø",
		"3": "Ù",
		"4": "Ú",
		"5": "Û",
		"6": "Ü",
		"7": "Ý",
		"8": "Þ",
		"9": "ß",
	}

	DECRYPT_MAPPING = map[string]string{}
)

func init() {
	for k, v := range ENCRYPT_MAPPING {
		DECRYPT_MAPPING[v] = k
	}
}

// Encrypt encrypts a readable string with the Kryptos algorithm
func Encrypt(input string) string {
	output := ""
	for _, char := range input {
		if val, ok := ENCRYPT_MAPPING[string(char)]; ok {
			output += val
		} else {
			output += string(char)
		}
	}
	return output
}

// Decrypt decrypts a string that was encrypted with the Kryptos algorithm
func Decrypt(input string) string {
	output := ""
	for _, char := range input {
		if val, ok := DECRYPT_MAPPING[string(char)]; ok {
			output += val
		} else {
			output += string(char)
		}
	}
	return output
}
<commit_msg>Golint fixes<commit_after>package kryptos

var (
	encryptMapping = map[string]string{
		"a": "¡",
		"b": "¢",
		"c": "£",
		"d": "¤",
		"e": "¥",
		"f": "¦",
		"g": "§",
		"h": "¨",
		"i": "ª",
		"j": "«",
		"k": "¬",
		"l": "®",
		"m": "¯",
		"n": "°",
		"o": "©",
		"p": "±",
		"q": "²",
		"r": "³",
		"s": "´",
		"t": "µ",
		"u": "¶",
		"v": "·",
		"w": "¸",
		"x": "¹",
		"y": "º",
		"z": "»",
		"A": "¼",
		"B": "½",
		"C": "¾",
		"D": "¿",
		"E": "À",
		"F": "Á",
		"G": "Â",
		"H": "Ã",
		"I": "Ä",
		"J": "Å",
		"K": "Æ",
		"L": "Ç",
		"M": "È",
		"N": "É",
		"O": "Ê",
		"P": "Ë",
		"Q": "Ì",
		"R": "Í",
		"S": "Î",
		"T": "Ï",
		"U": "Ð",
		"V": "Ñ",
		"W": "Ò",
		"X": "Ó",
		"Y": "Ô",
		"Z": "Õ",
		"0": "Ö",
		"1": "×",
		"2": "Ø",
		"3": "Ù",
		"4": "Ú",
		"5": "Û",
		"6": "Ü",
		"7": "Ý",
		"8": "Þ",
		"9": "ß",
	}

	decryptMapping = map[string]string{}
)

func init() {
	for k, v := range encryptMapping {
		decryptMapping[v] = k
	}
}

// Encrypt encrypts a readable string with the Kryptos algorithm
func Encrypt(input string) string {
	output := ""
	for _, char := range input {
		if val, ok := encryptMapping[string(char)]; ok {
			output += val
		} else {
			output += string(char)
		}
	}
	return output
}

// Decrypt decrypts a string that was encrypted with the Kryptos algorithm
func Decrypt(input string) string {
	output := ""
	for _, char := range input {
		if val, ok := decryptMapping[string(char)]; ok {
			output += val
		} else {
			output += string(char)
		}
	}
	return output
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/bborbe/backup/config"
	"github.com/bborbe/backup/dto"
	"github.com/bborbe/backup/service"
	"github.com/bborbe/backup/util"
	"github.com/bborbe/go/lock"
	"github.com/bborbe/log"
	"io"
	"os"
	"sort"
)

const LOCK_NAME = "/var/run/backup_clean.lock"

var logger = log.DefaultLogger

func main() {
	logLevelPtr := flag.Int("loglevel", config.DEFAULT_LOG_LEVEL, "int")
	rootdirPtr := flag.String("rootdir", config.DEFAULT_ROOT_DIR, "string")
	hostPtr := flag.String("host", config.DEFAULT_HOST, "string")
	flag.Parse()
	logger.SetLevelThreshold(*logLevelPtr)
	logger.Debugf("set log level to %s", log.LogLevelToString(*logLevelPtr))

	writer := os.Stdout
	logger.Debugf("use backup dir %s", *rootdirPtr)
	backupService := service.NewBackupService(*rootdirPtr)
	err := do(writer, backupService, *hostPtr, LOCK_NAME)
	if err != nil {
		logger.Fatal(err)
		os.Exit(1)
	}
}

func do(writer io.Writer, backupService service.BackupService, hostname string, lockName string) error {
	var err error
	var hosts []dto.Host

	l := lock.NewLock(lockName)
	err = l.Lock()
	if err != nil {
		return err
	}
	defer l.Unlock()
	logger.Debug("start")
	if hostname == config.DEFAULT_HOST {
		hosts, err = backupService.ListHosts()
		if err != nil {
			return err
		}
	} else {
		host, err := backupService.GetHost(hostname)
		if err != nil {
			return err
		}
		hosts = []dto.Host{host}
	}
	sort.Sort(util.HostByDate(hosts))
	for _, host := range hosts {
		err := backupService.Cleanup(host)
		if err != nil {
			return err
		}
		fmt.Fprintf(writer, "%s cleaned\n", host.GetName())
	}
	logger.Debug("done")
	return nil
}
<commit_msg>refactor lock  and update readme<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/bborbe/backup/config"
	"github.com/bborbe/backup/dto"
	"github.com/bborbe/backup/service"
	"github.com/bborbe/backup/util"
	"github.com/bborbe/lock"
	"github.com/bborbe/log"
	"io"
	"os"
	"sort"
)

const LOCK_NAME = "/var/run/backup_clean.lock"

var logger = log.DefaultLogger

func main() {
	logLevelPtr := flag.Int("loglevel", config.DEFAULT_LOG_LEVEL, "int")
	rootdirPtr := flag.String("rootdir", config.DEFAULT_ROOT_DIR, "string")
	hostPtr := flag.String("host", config.DEFAULT_HOST, "string")
	flag.Parse()
	logger.SetLevelThreshold(*logLevelPtr)
	logger.Debugf("set log level to %s", log.LogLevelToString(*logLevelPtr))

	writer := os.Stdout
	logger.Debugf("use backup dir %s", *rootdirPtr)
	backupService := service.NewBackupService(*rootdirPtr)
	err := do(writer, backupService, *hostPtr, LOCK_NAME)
	if err != nil {
		logger.Fatal(err)
		os.Exit(1)
	}
}

func do(writer io.Writer, backupService service.BackupService, hostname string, lockName string) error {
	var err error
	var hosts []dto.Host

	l := lock.NewLock(lockName)
	err = l.Lock()
	if err != nil {
		return err
	}
	defer l.Unlock()
	logger.Debug("start")
	if hostname == config.DEFAULT_HOST {
		hosts, err = backupService.ListHosts()
		if err != nil {
			return err
		}
	} else {
		host, err := backupService.GetHost(hostname)
		if err != nil {
			return err
		}
		hosts = []dto.Host{host}
	}
	sort.Sort(util.HostByDate(hosts))
	for _, host := range hosts {
		err := backupService.Cleanup(host)
		if err != nil {
			return err
		}
		fmt.Fprintf(writer, "%s cleaned\n", host.GetName())
	}
	logger.Debug("done")
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	// Stdlib
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"strings"
	"text/tabwriter"

	// Internal
	"github.com/salsita/salsaflow/config"
	"github.com/salsita/salsaflow/errs"
	"github.com/salsita/salsaflow/git"
	"github.com/salsita/salsaflow/log"
)

const (
	secretRemote = "AreYouWhoIThinkYouAreHuh"
	secretReply  = "IAmSalsaFlowHookYaDoofus!"
)

const zeroHash = "0000000000000000000000000000000000000000"

func main() {
	// `repo init` uses this secret check to see whether this hook is installed.
	if len(os.Args) == 2 && os.Args[1] == secretRemote {
		fmt.Println(secretReply)
		return
	}

	// The hook is always invoked as `pre-push <remote-name> <push-url>`.
	if len(os.Args) != 3 {
		panic(fmt.Errorf("argv: %#v", os.Args))
	}

	// Run the main function.
	msg := "Make sure the commits comply with the SalsaFlow requirements"
	log.Run(msg)
	if err := run(os.Args[1], os.Args[2]); err != nil {
		errs.LogFail(msg, err)
		fmt.Println()
		os.Exit(1)
	}
}

func run(remoteName, pushURL string) error {
	// The commits that are being pushed are listed on stdin.
	// The format is <local ref> <local sha1> <remote ref> <remote sha1>,
	// so we parse the input and collect all the local hexshas.
	msg := "Parse the hook input"
	var revRanges []string
	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		var (
			line  = scanner.Text()
			parts = strings.Split(line, " ")
		)
		if len(parts) != 4 {
			return errs.NewError(msg, nil, errors.New("invalid input line: "+line))
		}

		localSha, remoteSha := parts[1], parts[3]

		// Skip the refs that are being deleted.
		if localSha == zeroHash {
			continue
		}

		// Append the revision range for this input line.
		var revRange string
		if remoteSha == zeroHash {
			// In case we are pushing a new branch, check commits up to trunk.
			// There is probably no better guess that we can do in general.
			revRange = fmt.Sprintf("%s..%s", config.TrunkBranch, localSha)
		} else {
			// Otherwise check the commits that are new compared to the remote ref.
			revRange = fmt.Sprintf("%s..%s", remoteSha, localSha)
		}
		revRanges = append(revRanges, revRange)
	}
	if err := scanner.Err(); err != nil {
		return errs.NewError(msg, nil, err)
	}

	// Get the relevant commit objects.
	msg = "Get the commit objects to be pushed"
	var commits []*git.Commit
	for _, revRange := range revRanges {
		cs, stderr, err := git.ShowCommitRange(revRange)
		if err != nil {
			return errs.NewError(msg, stderr, err)
		}
		commits = append(commits, cs...)
	}

	// Validate the commit messages.
	msg = "Validate the commit messages"
	var invalid bool

	stderr := new(bytes.Buffer)
	tw := tabwriter.NewWriter(stderr, 0, 8, 4, '\t', 0)
	io.WriteString(tw, "\n")
	io.WriteString(tw, "Commit SHA\tError\n")
	io.WriteString(tw, "==========\t=====\n")

	for _, commit := range commits {
		if commit.Merge == "" {
			// Require the Change-Id tag in all non-merge commits.
			if commit.ChangeId == "" {
				fmt.Fprintf(tw, "%v\t%v\n", commit.SHA, "commit message: Change-Id tag missing")
				invalid = true
			}
			// Require the Story-Id tag in all non-merge commits.
			if commit.StoryId == "" {
				fmt.Fprintf(tw, "%v\t%v\n", commit.SHA, "commit message: Story-Id tag missing")
				invalid = true
			}
		}
	}

	if invalid {
		tw.Flush()
		stderr.WriteString("\n")
		return errs.NewError(msg, stderr, nil)
	}
	return nil
}
<commit_msg>hooks/pre-push: Check only the core branches<commit_after>package main

import (
	// Stdlib
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"strings"
	"text/tabwriter"

	// Internal
	"github.com/salsita/salsaflow/config"
	"github.com/salsita/salsaflow/errs"
	"github.com/salsita/salsaflow/git"
	"github.com/salsita/salsaflow/log"
)

const (
	secretRemote = "AreYouWhoIThinkYouAreHuh"
	secretReply  = "IAmSalsaFlowHookYaDoofus!"
)

const zeroHash = "0000000000000000000000000000000000000000"

func main() {
	// `repo init` uses this secret check to see whether this hook is installed.
	if len(os.Args) == 2 && os.Args[1] == secretRemote {
		fmt.Println(secretReply)
		return
	}

	// Tell the user what is happening.
	fmt.Println("---> Running the SalsaFlow pre-push hook")

	// The hook is always invoked as `pre-push <remote-name> <push-url>`.
	if len(os.Args) != 3 {
		log.Fatalf("Invalid arguments: %#v\n", os.Args)
	}

	// Run the main function.
	if err := run(os.Args[1], os.Args[2]); err != nil {
		errs.Log(err)
		fmt.Println()
		os.Exit(1)
	}
}

func run(remoteName, pushURL string) error {
	// Only check the project remote.
	if remoteName != config.OriginName {
		log.Log("Not pushing to the main project repository, check skipped")
		return nil
	}

	// The commits that are being pushed are listed on stdin.
	// The format is <local ref> <local sha1> <remote ref> <remote sha1>,
	// so we parse the input and collect all the local hexshas.
	var coreRefs = []string{
		"refs/heads/" + config.TrunkBranch,
		"refs/heads/" + config.ReleaseBranch,
		"refs/heads/" + config.ClientBranch,
		"refs/heads/" + config.MasterBranch,
	}

	msg := "Parse the hook input"
	var revRanges []string
	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		var (
			line  = scanner.Text()
			parts = strings.Split(line, " ")
		)
		if len(parts) != 4 {
			return errs.NewError(msg, nil, errors.New("invalid input line: "+line))
		}

		localSha, remoteRef, remoteSha := parts[1], parts[2], parts[3]

		// Skip the refs that are being deleted.
		if localSha == zeroHash {
			continue
		}

		// Check only updates to the core branches,
		// i.e. trunk, release, client or master.
		var checkReference bool
		for _, ref := range coreRefs {
			if remoteRef == ref {
				checkReference = true
			}
		}
		if !checkReference {
			continue
		}

		log.Log(fmt.Sprintf("Checking commits updating remote reference '%s'", remoteRef))

		// Append the revision range for this input line.
		var revRange string
		if remoteSha == zeroHash {
			// In case we are pushing a new branch, check commits up to trunk.
			// There is probably no better guess that we can do in general.
			revRange = fmt.Sprintf("%s..%s", config.TrunkBranch, localSha)
		} else {
			// Otherwise check the commits that are new compared to the remote ref.
			revRange = fmt.Sprintf("%s..%s", remoteSha, localSha)
		}
		revRanges = append(revRanges, revRange)
	}
	if err := scanner.Err(); err != nil {
		return errs.NewError(msg, nil, err)
	}

	// Get the relevant commit objects.
	msg = "Get the commit objects to be pushed"
	var commits []*git.Commit
	for _, revRange := range revRanges {
		cs, stderr, err := git.ShowCommitRange(revRange)
		if err != nil {
			return errs.NewError(msg, stderr, err)
		}
		commits = append(commits, cs...)
	}

	// Validate the commit messages.
	msg = "Validate the commit messages"
	var invalid bool

	stderr := new(bytes.Buffer)
	tw := tabwriter.NewWriter(stderr, 0, 8, 4, '\t', 0)
	io.WriteString(tw, "\n")
	io.WriteString(tw, "Commit SHA\tError\n")
	io.WriteString(tw, "==========\t=====\n")

	for _, commit := range commits {
		if commit.Merge == "" {
			// Require the Change-Id tag in all non-merge commits.
			if commit.ChangeId == "" {
				fmt.Fprintf(tw, "%v\t%v\n", commit.SHA, "commit message: Change-Id tag missing")
				invalid = true
			}
			// Require the Story-Id tag in all non-merge commits.
			if commit.StoryId == "" {
				fmt.Fprintf(tw, "%v\t%v\n", commit.SHA, "commit message: Story-Id tag missing")
				invalid = true
			}
		}
	}

	if invalid {
		tw.Flush()
		stderr.WriteString("\n")
		return errs.NewError(msg, stderr, nil)
	}
	return nil
}
<|endoftext|>


<commit_before>package updates

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"
	"time"

	"github.com/hashicorp/go-version"
	"github.com/pkg/errors"
	"github.com/yext/edward/common"
)

func UpdateAvailable(repo, currentVersion, cachePath string, logger common.Logger) (bool, string, error) {
	output, err := exec.Command("git", "ls-remote", "-t", "git://"+repo).CombinedOutput()
	if err != nil {
		return false, "", errors.WithStack(err)
	}

	printf(logger, "Checking for cached version at %v", cachePath)
	isCached, latestVersion, err := getCachedVersion(cachePath)
	if err != nil {
		return false, "", errors.WithStack(err)
	}

	if !isCached {
		printf(logger, "No cached version, requesting from Git\n")
		latestVersion, err = findLatestVersionTag(output)
		if err != nil {
			return false, "", errors.WithStack(err)
		}
		printf(logger, "Caching version: %v", latestVersion)
		err = cacheVersion(cachePath, latestVersion)
		if err != nil {
			return false, "", errors.WithStack(err)
		}
	} else {
		printf(logger, "Found cached version\n")
	}

	printf(logger, "Comparing latest release %v, to current version %v\n", latestVersion, currentVersion)

	lv, err1 := version.NewVersion(latestVersion)
	cv, err2 := version.NewVersion(currentVersion)

	if err1 != nil {
		return false, latestVersion, errors.WithStack(err)
	}
	if err2 != nil {
		return true, latestVersion, errors.WithStack(err)
	}

	return cv.LessThan(lv), latestVersion, nil
}

func findLatestVersionTag(refs []byte) (string, error) {
	r := bytes.NewReader(refs)
	reader := bufio.NewReader(r)
	line, isPrefix, err := reader.ReadLine()

	var greatestVersion string

	var validID = regexp.MustCompile(`([0-9]+\.[0-9]+\.[0-9])?$`)
	for err != io.EOF {
		if isPrefix {
			fmt.Println("Prefix")
		}
		match := validID.FindString(string(line))
		if len(match) > 0 && match > greatestVersion {
			greatestVersion = match
		}
		line, isPrefix, err = reader.ReadLine()
	}
	return greatestVersion, nil
}

func getCachedVersion(cachePath string) (wasCached bool, cachedVersion string, err error) {
	info, err := os.Stat(cachePath)
	if err != nil {
		if os.IsNotExist(err) {
			return false, "", nil
		}
		return false, "", errors.WithStack(err)
	}
	duration := time.Since(info.ModTime())
	if duration.Hours() >= 1 {
		return false, "", nil
	}
	content, err := ioutil.ReadFile(cachePath)
	if err != nil {
		return false, "", errors.WithStack(err)
	}
	return true, string(content), nil
}

func cacheVersion(cachePath, versionToCache string) error {
	err := ioutil.WriteFile(cachePath, []byte(versionToCache), 0644)
	return errors.WithStack(err)
}

func printf(logger common.Logger, f string, v ...interface{}) {
	if logger != nil {
		logger.Printf(f, v...)
	}
}
<commit_msg>Clean up a warning in updates package.<commit_after>package updates

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"
	"time"

	"github.com/hashicorp/go-version"
	"github.com/pkg/errors"
	"github.com/yext/edward/common"
)

// UpdateAvailable determines if a newer version is available given a repo
func UpdateAvailable(repo, currentVersion, cachePath string, logger common.Logger) (bool, string, error) {
	output, err := exec.Command("git", "ls-remote", "-t", "git://"+repo).CombinedOutput()
	if err != nil {
		return false, "", errors.WithStack(err)
	}

	printf(logger, "Checking for cached version at %v", cachePath)
	isCached, latestVersion, err := getCachedVersion(cachePath)
	if err != nil {
		return false, "", errors.WithStack(err)
	}

	if !isCached {
		printf(logger, "No cached version, requesting from Git\n")
		latestVersion, err = findLatestVersionTag(output)
		if err != nil {
			return false, "", errors.WithStack(err)
		}
		printf(logger, "Caching version: %v", latestVersion)
		err = cacheVersion(cachePath, latestVersion)
		if err != nil {
			return false, "", errors.WithStack(err)
		}
	} else {
		printf(logger, "Found cached version\n")
	}

	printf(logger, "Comparing latest release %v, to current version %v\n", latestVersion, currentVersion)

	lv, err1 := version.NewVersion(latestVersion)
	cv, err2 := version.NewVersion(currentVersion)

	if err1 != nil {
		return false, latestVersion, errors.WithStack(err)
	}
	if err2 != nil {
		return true, latestVersion, errors.WithStack(err)
	}

	return cv.LessThan(lv), latestVersion, nil
}

func findLatestVersionTag(refs []byte) (string, error) {
	r := bytes.NewReader(refs)
	reader := bufio.NewReader(r)
	line, isPrefix, err := reader.ReadLine()

	var greatestVersion string

	var validID = regexp.MustCompile(`([0-9]+\.[0-9]+\.[0-9])?$`)
	for err != io.EOF {
		if isPrefix {
			fmt.Println("Prefix")
		}
		match := validID.FindString(string(line))
		if len(match) > 0 && match > greatestVersion {
			greatestVersion = match
		}
		line, isPrefix, err = reader.ReadLine()
	}
	return greatestVersion, nil
}

func getCachedVersion(cachePath string) (wasCached bool, cachedVersion string, err error) {
	info, err := os.Stat(cachePath)
	if err != nil {
		if os.IsNotExist(err) {
			return false, "", nil
		}
		return false, "", errors.WithStack(err)
	}
	duration := time.Since(info.ModTime())
	if duration.Hours() >= 1 {
		return false, "", nil
	}
	content, err := ioutil.ReadFile(cachePath)
	if err != nil {
		return false, "", errors.WithStack(err)
	}
	return true, string(content), nil
}

func cacheVersion(cachePath, versionToCache string) error {
	err := ioutil.WriteFile(cachePath, []byte(versionToCache), 0644)
	return errors.WithStack(err)
}

func printf(logger common.Logger, f string, v ...interface{}) {
	if logger != nil {
		logger.Printf(f, v...)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	_ "log"
	"math"
	"strings"
	"time"
)

type Frame string

func (f Frame) String() string {
	return string(f)
}

func trimAt(s string, pos int) string {
	t := []rune(s)
	n := len(t)

	if n > pos {
		return string(t[0:pos])
	} else {
		// Pad with blanks
		return s + strings.Repeat(" ", pos-n)
	}
}

func frameLine(s string, chr rune) string {
	t := []rune(s)

	// Replace first
	t[0] = chr

	// Replace last
	t[len(t)-1] = chr

	return string(t)
}

// Frame lines
var (
	top  = "┌——————————————————————————————————————————————————————————————┬———————————————┐"
	side = '│'
	bttm = "└——————————————————————————————————————————————————————————————————————————————┘"
)

func MakeFrame(pic []rune) Frame {
	lines := strings.Split(string(pic), "\n")

	frameWidth := len([]rune(top))

	// Frame each line
	for i, _ := range lines {
		// Make sure line is frameWidth columns long
		nl := trimAt(lines[i], frameWidth)

		// Blit with side glyphs
		nl = frameLine(nl, side)

		// Special case for line 4
		if i == 4 {
			t := []rune(nl)
			t[79] = '┤'
			nl = string(t)
		}

		lines[i] = nl
	}

	// Add top frame line
	lines = append([]string{top}, lines...) // Poor man's push
	// Add bottom frame line
	lines = append(lines, bttm)

	return Frame(strings.Join(lines, "\n"))
}

func _f(b []rune, ch rune) int {
	max := len(b)
	i := 0
	for ; i < max; i++ {
		if b[i] == ch {
			break
		}
	}
	return i
}
func splice(b []rune, yidx, offs int, val []rune) {
	maxx := _f(b, '\n') + 1
	for i, v := range val {
		if i+offs > maxx {
			break
		}
		b[yidx*maxx+offs+i] = v
	}
}

func normalizeDegrees(degree float64) float64 {
	periods := int(degree / 360)
	return degree - float64(periods*360)
}

const radToDegree = 180 / math.Pi

func donut(aspectA, aspectB float64, frameDelay time.Duration, stream chan<- Frame) {
	var A, B float64 = aspectA, aspectB

	// Frame
	b := make([]rune, 80*22)

	// Torus matrix with light cooficients
	z := make([]float64, len(b))

	// stats
	var fps float64  // frames per second
	frameN := 0      // frame count
	t0 := time.Now() // start time

	// Forever
	for {
		// Rotational parameters
		A += 0.07 // Yaw  - rotation
		B += 0.03 // Roll - rotation

		sA, cA := math.Sincos(A)
		sB, cB := math.Sincos(B)

		// Blank frame
		for k := 0; k < len(b); k++ {
			if k%80 == 79 {
				b[k] = '\n'
			} else {
				b[k] = ' '
			}
			z[k] = 0 // Zero-out torus matrix
		}

		// Draw torus with rotational aspect

		// x(theta, phy) = (R + r * cos(phy) * cos(theta)
		// y(theta, phy) = (R + r * cos(phy) * sin(theta)
		// z(theta, phy) = r * sin(phy)

		// theta, phy - angles from 0 to 2 * PI
		// R          - major radius
		// r          - minor radius

		// Theta
		for j := float64(0); j < 6.28; j += 0.07 {
			// sin and cos of theta
			st, ct := math.Sincos(j)

			// Phy
			for i := float64(0); i < 6.28; i += 0.02 {

				// Unknown =  2 + cos(theta)
				h := ct + 2

				// sin and cos of phy
				sp, cp := math.Sincos(i)

				// Unknown =  1 / [ sin(phy) * (2 + cos(theta)) * sin(A) + sin(theta) * cos(A) + 5 ]
				// guess - value for our torus matrix as it rotates in 3-D space
				D := 1 / (sp*h*sA + st*cA + 5)

				// Unknown =  sin(phy) * (2 + cos(theta)) * cos(A) - sin(theta) * sin(A)
				t := sp*h*cA - st*sA

				// X-axis coordinate
				x := int(40 + 30*D*(cp*h*cB-t*sB))

				// Y-axis coordinate
				y := int(12 + 15*D*(cp*h*sB+t*cB))

				// Frame pixel index
				o := x + 80*y

				// Light coeficient (grey-scale index)
				N := int(8 * ((st*sA-sp*ct*cA)*cB - sp*ct*sA - st*cA - cp*ct*sB))

				// Draw if we are within frame and pixel is visible
				if y < 22 && y >= 0 && x >= 0 && x < 79 && D > z[o] {
					// Fill
					z[o] = D
					n := 0
					if N > 0 {
						n = N
					}

					// Fill the pixel on the canvas (frame)
					b[o] = []rune(".,-~:;=!*#$@")[n]
					// b[o] = []rune("∙◦▪●☼◊≠≡☺♦☻◙")[n]
				}
			}
		}
		// Count frames
		frameN++

		// Calculate frame per second rate
		fps = float64(frameN) / time.Since(t0).Seconds()

		// display stats
		var (
			frame = []rune(fmt.Sprintf("│ Frame: %5d", frameN))
			rate  = []rune(fmt.Sprintf("│   FPS: %5.1f", fps))
			roll  = []rune(fmt.Sprintf("│  Roll: %5.1f˚", normalizeDegrees(B*radToDegree)))
			yaw   = []rune(fmt.Sprintf("│   Yaw: %5.1f˚", normalizeDegrees(A*radToDegree)))
			bttm  = []rune("└———————————————")
			offs  = 77 - len(frame)
		)
		splice(b, 0, offs, frame)
		splice(b, 1, offs, rate)
		splice(b, 2, offs, roll)
		splice(b, 3, offs, yaw)
		splice(b, 4, offs, bttm)

		// Send frame
		stream <- MakeFrame(b)
	}
}

func genFrameStream(f func(float64, float64, time.Duration, chan<- Frame)) <-chan Frame {
	stream := make(chan Frame)

	go func() {
		f(0, 0, time.Millisecond*40, stream)
	}()

	return stream
}

func main() {
	frames := genFrameStream(donut)
	timeout := time.After(5 * time.Second)

	for {
		select {
		case <-timeout:
			return
		case frame := <-frames:
			fmt.Print("\033[H\033[2J")        // Clear screen
			fmt.Print("\x0c", frame, "\n")    // Print frame
			time.Sleep(40 * time.Millisecond) // Delay between frames
		}
	}
}
<commit_msg>Make frame channel buffered<commit_after>package main

import (
	"fmt"
	_ "log"
	"math"
	"strings"
	"time"
)

type Frame string

func (f Frame) String() string {
	return string(f)
}

func trimAt(s string, pos int) string {
	t := []rune(s)
	n := len(t)

	if n > pos {
		return string(t[0:pos])
	} else {
		// Pad with blanks
		return s + strings.Repeat(" ", pos-n)
	}
}

func frameLine(s string, chr rune) string {
	t := []rune(s)

	// Replace first
	t[0] = chr

	// Replace last
	t[len(t)-1] = chr

	return string(t)
}

// Frame lines
var (
	top  = "┌——————————————————————————————————————————————————————————————┬———————————————┐"
	side = '│'
	bttm = "└——————————————————————————————————————————————————————————————————————————————┘"
)

func MakeFrame(pic []rune) Frame {
	lines := strings.Split(string(pic), "\n")

	frameWidth := len([]rune(top))

	// Frame each line
	for i, _ := range lines {
		// Make sure line is frameWidth columns long
		nl := trimAt(lines[i], frameWidth)

		// Blit with side glyphs
		nl = frameLine(nl, side)

		// Special case for line 4
		if i == 4 {
			t := []rune(nl)
			t[79] = '┤'
			nl = string(t)
		}

		lines[i] = nl
	}

	// Add top frame line
	lines = append([]string{top}, lines...) // Poor man's push
	// Add bottom frame line
	lines = append(lines, bttm)

	return Frame(strings.Join(lines, "\n"))
}

func _f(b []rune, ch rune) int {
	max := len(b)
	i := 0
	for ; i < max; i++ {
		if b[i] == ch {
			break
		}
	}
	return i
}
func splice(b []rune, yidx, offs int, val []rune) {
	maxx := _f(b, '\n') + 1
	for i, v := range val {
		if i+offs > maxx {
			break
		}
		b[yidx*maxx+offs+i] = v
	}
}

func normalizeDegrees(degree float64) float64 {
	periods := int(degree / 360)
	return degree - float64(periods*360)
}

const radToDegree = 180 / math.Pi

func donut(aspectA, aspectB float64, frameDelay time.Duration, stream chan<- Frame) {
	var A, B float64 = aspectA, aspectB

	// Frame
	b := make([]rune, 80*22)

	// Torus matrix with light cooficients
	z := make([]float64, len(b))

	// stats
	var fps float64  // frames per second
	frameN := 0      // frame count
	t0 := time.Now() // start time

	// Forever
	for {
		// Rotational parameters
		A += 0.07 // Yaw  - rotation
		B += 0.03 // Roll - rotation

		sA, cA := math.Sincos(A)
		sB, cB := math.Sincos(B)

		// Blank frame
		for k := 0; k < len(b); k++ {
			if k%80 == 79 {
				b[k] = '\n'
			} else {
				b[k] = ' '
			}
			z[k] = 0 // Zero-out torus matrix
		}

		// Draw torus with rotational aspect

		// x(theta, phy) = (R + r * cos(phy) * cos(theta)
		// y(theta, phy) = (R + r * cos(phy) * sin(theta)
		// z(theta, phy) = r * sin(phy)

		// theta, phy - angles from 0 to 2 * PI
		// R          - major radius
		// r          - minor radius

		// Theta
		for j := float64(0); j < 6.28; j += 0.07 {
			// sin and cos of theta
			st, ct := math.Sincos(j)

			// Phy
			for i := float64(0); i < 6.28; i += 0.02 {

				// Unknown =  2 + cos(theta)
				h := ct + 2

				// sin and cos of phy
				sp, cp := math.Sincos(i)

				// Unknown =  1 / [ sin(phy) * (2 + cos(theta)) * sin(A) + sin(theta) * cos(A) + 5 ]
				// guess - value for our torus matrix as it rotates in 3-D space
				D := 1 / (sp*h*sA + st*cA + 5)

				// Unknown =  sin(phy) * (2 + cos(theta)) * cos(A) - sin(theta) * sin(A)
				t := sp*h*cA - st*sA

				// X-axis coordinate
				x := int(40 + 30*D*(cp*h*cB-t*sB))

				// Y-axis coordinate
				y := int(12 + 15*D*(cp*h*sB+t*cB))

				// Frame pixel index
				o := x + 80*y

				// Light coeficient (grey-scale index)
				N := int(8 * ((st*sA-sp*ct*cA)*cB - sp*ct*sA - st*cA - cp*ct*sB))

				// Draw if we are within frame and pixel is visible
				if y < 22 && y >= 0 && x >= 0 && x < 79 && D > z[o] {
					// Fill
					z[o] = D
					n := 0
					if N > 0 {
						n = N
					}

					// Fill the pixel on the canvas (frame)
					b[o] = []rune(".,-~:;=!*#$@")[n]
					// b[o] = []rune("∙◦▪●☼◊≠≡☺♦☻◙")[n]
				}
			}
		}
		// Count frames
		frameN++

		// Calculate frame per second rate
		fps = float64(frameN) / time.Since(t0).Seconds()

		// display stats
		var (
			frame = []rune(fmt.Sprintf("│ Frame: %5d", frameN))
			rate  = []rune(fmt.Sprintf("│   FPS: %5.1f", fps))
			roll  = []rune(fmt.Sprintf("│  Roll: %5.1f˚", normalizeDegrees(B*radToDegree)))
			yaw   = []rune(fmt.Sprintf("│   Yaw: %5.1f˚", normalizeDegrees(A*radToDegree)))
			bttm  = []rune("└———————————————")
			offs  = 77 - len(frame)
		)
		splice(b, 0, offs, frame)
		splice(b, 1, offs, rate)
		splice(b, 2, offs, roll)
		splice(b, 3, offs, yaw)
		splice(b, 4, offs, bttm)

		// Send frame
		stream <- MakeFrame(b)
	}
}

func genFrameStream(f func(float64, float64, time.Duration, chan<- Frame)) <-chan Frame {
	stream := make(chan Frame, 1) // Always a room for the next frame

	go func() {
		f(0, 0, time.Millisecond*40, stream)
	}()

	return stream
}

func main() {
	frames := genFrameStream(donut)
	timeout := time.After(5 * time.Second)

	for {
		select {
		case <-timeout:
			return
		case frame := <-frames:
			fmt.Print("\033[H\033[2J")        // Clear screen
			fmt.Print("\x0c", frame, "\n")    // Print frame
			time.Sleep(40 * time.Millisecond) // Delay between frames
		}
	}
}
<|endoftext|>