<commit_before>package gateway

import (
	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/fastrand"
)

// managedPeerManagerConnect is a blocking function which tries to connect to
// the input addreess as a peer.
func (g *Gateway) managedPeerManagerConnect(addr modules.NetAddress) {
	g.log.Debugf("[PMC] [%v] Attempting connection", addr)
	err := g.managedConnect(addr)
	if err == errPeerExists {
		// This peer is already connected to us. Safety around the
		// oubound peers relates to the fact that we have picked out
		// the outbound peers instead of allow the attacker to pick out
		// the peers for us. Because we have made the selection, it is
		// okay to set the peer as an outbound peer.
		//
		// The nodelist size check ensures that an attacker can't flood
		// a new node with a bunch of inbound requests. Doing so would
		// result in a nodelist that's entirely full of attacker nodes.
		// There's not much we can do about that anyway, but at least
		// we can hold off making attacker nodes 'outbound' peers until
		// our nodelist has had time to fill up naturally.
		g.mu.Lock()
		p, exists := g.peers[addr]
		if exists {
			// Have to check it exists because we released the lock, a
			// race condition could mean that the peer was disconnected
			// before this code block was reached.
			p.Inbound = false
			if n, ok := g.nodes[p.NetAddress]; ok && !n.WasOutboundPeer {
				n.WasOutboundPeer = true
				g.nodes[n.NetAddress] = n
				g.saveSync()
			}
			g.log.Debugf("[PMC] [SUCCESS] [%v] existing peer has been converted to outbound peer", addr)
		}
		g.mu.Unlock()
	} else if err != nil {
		g.log.Debugf("[PMC] [ERROR] [%v] WARN: removing peer because automatic connect failed: %v\n", addr, err)

		// Remove the node, but only if there are enough nodes in the node list.
		g.mu.Lock()
		if len(g.nodes) > pruneNodeListLen {
			g.removeNode(addr)
		}
		g.mu.Unlock()
	} else {
		g.log.Debugf("[PMC] [SUCCESS] [%v] peer successfully added", addr)
	}
}

// numOutboundPeers returns the number of outbound peers in the gateway.
func (g *Gateway) numOutboundPeers() int {
	n := 0
	for _, p := range g.peers {
		if !p.Inbound {
			n++
		}
	}
	return n
}

// permanentPeerManager tries to keep the Gateway well-connected. As long as
// the Gateway is not well-connected, it tries to connect to random nodes.
func (g *Gateway) permanentPeerManager(closedChan chan struct{}) {
	// Send a signal upon shutdown.
	defer close(closedChan)
	defer g.log.Debugln("INFO: [PPM] Permanent peer manager is shutting down")

	// permanentPeerManager will attempt to connect to peers asynchronously,
	// such that multiple connection attempts can be open at once, but a
	// limited number.
	connectionLimiterChan := make(chan struct{}, maxConcurrentOutboundPeerRequests)

	g.log.Debugln("INFO: [PPM] Permanent peer manager has started")

	for {
		// Fetch the set of nodes to try.
		g.mu.RLock()
		nodes := g.buildPeerManagerNodeList()
		g.mu.RUnlock()
		if len(nodes) == 0 {
			g.log.Debugln("[PPM] Node list is empty, sleeping")
			if !g.managedSleep(noNodesDelay) {
				return
			}
			continue
		}

		for _, addr := range nodes {
			// Break as soon as we have enough outbound peers.
			g.mu.RLock()
			numOutboundPeers := g.numOutboundPeers()
			isOutboundPeer := g.peers[addr] != nil && !g.peers[addr].Inbound
			g.mu.RUnlock()
			if numOutboundPeers >= wellConnectedThreshold {
				g.log.Debugln("INFO: [PPM] Gateway has enough peers, sleeping.")
				if !g.managedSleep(wellConnectedDelay) {
					return
				}
				break
			}
			if isOutboundPeer {
				// Skip current outbound peers.
				continue
			}

			g.log.Debugln("[PPM] Fetched a random node:", addr)

			// We need at least some of our outbound peers to be remote peers. If
			// we already have reached a certain threshold of outbound peers and
			// this peer is a local peer, do not consider it for an outbound peer.
			// Sleep briefly to prevent the gateway from hogging the CPU if all
			// peers are local.
			if numOutboundPeers >= maxLocalOutboundPeers && addr.IsLocal() && build.Release != "testing" {
				g.log.Debugln("[PPM] Ignorning selected peer; this peer is local and we already have multiple outbound peers:", addr)
				if !g.managedSleep(unwantedLocalPeerDelay) {
					return
				}
				continue
			}

			// Try connecting to that peer in a goroutine. Do not block unless
			// there are currently 3 or more peer connection attempts open at once.
			// Before spawning the thread, make sure that there is enough room by
			// throwing a struct into the buffered channel.
			g.log.Debugln("[PPM] Trying to connect to a node:", addr)
			connectionLimiterChan <- struct{}{}
			go func(addr modules.NetAddress) {
				// After completion, take the struct out of the channel so that the
				// next thread may proceed.
				defer func() {
					<-connectionLimiterChan
				}()

				if err := g.threads.Add(); err != nil {
					return
				}
				defer g.threads.Done()
				// peerManagerConnect will handle all of its own logging.
				g.managedPeerManagerConnect(addr)
			}(addr)

			// Wait a bit before trying the next peer. The peer connections are
			// non-blocking, so they should be spaced out to avoid spinning up an
			// uncontrolled number of threads and therefore peer connections.
			if !g.managedSleep(acquiringPeersDelay) {
				return
			}
		}
	}
}

// buildPeerManagerNodeList returns the gateway's node list in the order that
// permanentPeerManager should attempt to connect to them.
func (g *Gateway) buildPeerManagerNodeList() []modules.NetAddress {
	// flatten the node map, inserting in random order
	nodes := make([]modules.NetAddress, len(g.nodes))
	perm := fastrand.Perm(len(nodes))
	for _, node := range g.nodes {
		nodes[perm[0]] = node.NetAddress
		perm = perm[1:]
	}

	// swap the outbound nodes to the front of the list
	numOutbound := 0
	for i, node := range nodes {
		if g.nodes[node].WasOutboundPeer {
			nodes[numOutbound], nodes[i] = nodes[i], nodes[numOutbound]
			numOutbound++
		}
	}
	return nodes
}
<commit_msg>drop saveSync from managedPeerManagerConnect<commit_after>package gateway

import (
	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/fastrand"
)

// managedPeerManagerConnect is a blocking function which tries to connect to
// the input addreess as a peer.
func (g *Gateway) managedPeerManagerConnect(addr modules.NetAddress) {
	g.log.Debugf("[PMC] [%v] Attempting connection", addr)
	err := g.managedConnect(addr)
	if err == errPeerExists {
		// This peer is already connected to us. Safety around the
		// oubound peers relates to the fact that we have picked out
		// the outbound peers instead of allow the attacker to pick out
		// the peers for us. Because we have made the selection, it is
		// okay to set the peer as an outbound peer.
		//
		// The nodelist size check ensures that an attacker can't flood
		// a new node with a bunch of inbound requests. Doing so would
		// result in a nodelist that's entirely full of attacker nodes.
		// There's not much we can do about that anyway, but at least
		// we can hold off making attacker nodes 'outbound' peers until
		// our nodelist has had time to fill up naturally.
		g.mu.Lock()
		p, exists := g.peers[addr]
		if exists {
			// Have to check it exists because we released the lock, a
			// race condition could mean that the peer was disconnected
			// before this code block was reached.
			p.Inbound = false
			if n, ok := g.nodes[p.NetAddress]; ok && !n.WasOutboundPeer {
				n.WasOutboundPeer = true
				g.nodes[n.NetAddress] = n
			}
			g.log.Debugf("[PMC] [SUCCESS] [%v] existing peer has been converted to outbound peer", addr)
		}
		g.mu.Unlock()
	} else if err != nil {
		g.log.Debugf("[PMC] [ERROR] [%v] WARN: removing peer because automatic connect failed: %v\n", addr, err)

		// Remove the node, but only if there are enough nodes in the node list.
		g.mu.Lock()
		if len(g.nodes) > pruneNodeListLen {
			g.removeNode(addr)
		}
		g.mu.Unlock()
	} else {
		g.log.Debugf("[PMC] [SUCCESS] [%v] peer successfully added", addr)
	}
}

// numOutboundPeers returns the number of outbound peers in the gateway.
func (g *Gateway) numOutboundPeers() int {
	n := 0
	for _, p := range g.peers {
		if !p.Inbound {
			n++
		}
	}
	return n
}

// permanentPeerManager tries to keep the Gateway well-connected. As long as
// the Gateway is not well-connected, it tries to connect to random nodes.
func (g *Gateway) permanentPeerManager(closedChan chan struct{}) {
	// Send a signal upon shutdown.
	defer close(closedChan)
	defer g.log.Debugln("INFO: [PPM] Permanent peer manager is shutting down")

	// permanentPeerManager will attempt to connect to peers asynchronously,
	// such that multiple connection attempts can be open at once, but a
	// limited number.
	connectionLimiterChan := make(chan struct{}, maxConcurrentOutboundPeerRequests)

	g.log.Debugln("INFO: [PPM] Permanent peer manager has started")

	for {
		// Fetch the set of nodes to try.
		g.mu.RLock()
		nodes := g.buildPeerManagerNodeList()
		g.mu.RUnlock()
		if len(nodes) == 0 {
			g.log.Debugln("[PPM] Node list is empty, sleeping")
			if !g.managedSleep(noNodesDelay) {
				return
			}
			continue
		}

		for _, addr := range nodes {
			// Break as soon as we have enough outbound peers.
			g.mu.RLock()
			numOutboundPeers := g.numOutboundPeers()
			isOutboundPeer := g.peers[addr] != nil && !g.peers[addr].Inbound
			g.mu.RUnlock()
			if numOutboundPeers >= wellConnectedThreshold {
				g.log.Debugln("INFO: [PPM] Gateway has enough peers, sleeping.")
				if !g.managedSleep(wellConnectedDelay) {
					return
				}
				break
			}
			if isOutboundPeer {
				// Skip current outbound peers.
				continue
			}

			g.log.Debugln("[PPM] Fetched a random node:", addr)

			// We need at least some of our outbound peers to be remote peers. If
			// we already have reached a certain threshold of outbound peers and
			// this peer is a local peer, do not consider it for an outbound peer.
			// Sleep briefly to prevent the gateway from hogging the CPU if all
			// peers are local.
			if numOutboundPeers >= maxLocalOutboundPeers && addr.IsLocal() && build.Release != "testing" {
				g.log.Debugln("[PPM] Ignorning selected peer; this peer is local and we already have multiple outbound peers:", addr)
				if !g.managedSleep(unwantedLocalPeerDelay) {
					return
				}
				continue
			}

			// Try connecting to that peer in a goroutine. Do not block unless
			// there are currently 3 or more peer connection attempts open at once.
			// Before spawning the thread, make sure that there is enough room by
			// throwing a struct into the buffered channel.
			g.log.Debugln("[PPM] Trying to connect to a node:", addr)
			connectionLimiterChan <- struct{}{}
			go func(addr modules.NetAddress) {
				// After completion, take the struct out of the channel so that the
				// next thread may proceed.
				defer func() {
					<-connectionLimiterChan
				}()

				if err := g.threads.Add(); err != nil {
					return
				}
				defer g.threads.Done()
				// peerManagerConnect will handle all of its own logging.
				g.managedPeerManagerConnect(addr)
			}(addr)

			// Wait a bit before trying the next peer. The peer connections are
			// non-blocking, so they should be spaced out to avoid spinning up an
			// uncontrolled number of threads and therefore peer connections.
			if !g.managedSleep(acquiringPeersDelay) {
				return
			}
		}
	}
}

// buildPeerManagerNodeList returns the gateway's node list in the order that
// permanentPeerManager should attempt to connect to them.
func (g *Gateway) buildPeerManagerNodeList() []modules.NetAddress {
	// flatten the node map, inserting in random order
	nodes := make([]modules.NetAddress, len(g.nodes))
	perm := fastrand.Perm(len(nodes))
	for _, node := range g.nodes {
		nodes[perm[0]] = node.NetAddress
		perm = perm[1:]
	}

	// swap the outbound nodes to the front of the list
	numOutbound := 0
	for i, node := range nodes {
		if g.nodes[node].WasOutboundPeer {
			nodes[numOutbound], nodes[i] = nodes[i], nodes[numOutbound]
			numOutbound++
		}
	}
	return nodes
}
<|endoftext|>


<commit_before>package kontainerdriver

import (
	"fmt"

	"github.com/rancher/norman/types"
	v3 "github.com/rancher/types/apis/management.cattle.io/v3"
	client "github.com/rancher/types/client/management/v3"
	"github.com/rancher/types/config"
	"github.com/sirupsen/logrus"
	"k8s.io/client-go/tools/cache"
)

type Format struct {
	ClusterIndexer cache.Indexer
}

func NewFormatter(manangement *config.ScaledContext) types.Formatter {
	clusterInformer := manangement.Management.Clusters("").Controller().Informer()
	// use an indexer instead of expensive k8s api calls
	clusterInformer.AddIndexers(map[string]cache.IndexFunc{
		clusterByGenericEngineConfigKey: clusterByKontainerDriver,
	})

	format := Format{
		ClusterIndexer: clusterInformer.GetIndexer(),
	}
	return format.Formatter
}

func CollectionFormatter(apiContext *types.APIContext, collection *types.GenericCollection) {
	collection.AddAction(apiContext, "refresh")
	collection.Links["rancher-images"] = fmt.Sprintf("%srancher-images", apiContext.URLBuilder.Current())
}

const clusterByGenericEngineConfigKey = "genericEngineConfig"

// clusterByKontainerDriver is an indexer function that uses the cluster genericEngineConfig
// driverName field
func clusterByKontainerDriver(obj interface{}) ([]string, error) {
	cluster, ok := obj.(*v3.Cluster)
	if !ok {
		return []string{}, nil
	}
	engineConfig := cluster.Spec.GenericEngineConfig
	if engineConfig == nil {
		return []string{}, nil
	}
	driverName, ok := (*engineConfig)["driverName"].(string)
	if !ok {
		return []string{}, nil
	}

	return []string{driverName}, nil
}

func (f *Format) Formatter(request *types.APIContext, resource *types.RawResource) {
	state, ok := resource.Values["state"].(string)
	if ok {
		if state == "active" {
			resource.AddAction(request, "deactivate")
		}

		if state == "inactive" {
			resource.AddAction(request, "activate")
		}
	}
	// if cluster driver is a built-in, delete removal link from UI
	if builtIn, _ := resource.Values[client.KontainerDriverFieldBuiltIn].(bool); builtIn {
		delete(resource.Links, "remove")
		return
	}
	resName := resource.Values["id"]
	// resName will be nil when first added
	if resName != nil {
		clustersWithKontainerDriver, err := f.ClusterIndexer.ByIndex(clusterByGenericEngineConfigKey, resName.(string))
		if err != nil {
			logrus.Warnf("failed to determine if kontainer driver %v was in use by a cluster : %v", resName.(string), err)
		} else if len(clustersWithKontainerDriver) != 0 {
			// if cluster driver in use, delete removal link from UI
			delete(resource.Links, "remove")
		}
	}
}
<commit_msg>build link to rancher-images correctly<commit_after>package kontainerdriver

import (
	"fmt"
	"strings"

	"github.com/rancher/norman/types"
	v3 "github.com/rancher/types/apis/management.cattle.io/v3"
	client "github.com/rancher/types/client/management/v3"
	"github.com/rancher/types/config"
	"github.com/sirupsen/logrus"
	"k8s.io/client-go/tools/cache"
)

type Format struct {
	ClusterIndexer cache.Indexer
}

func NewFormatter(manangement *config.ScaledContext) types.Formatter {
	clusterInformer := manangement.Management.Clusters("").Controller().Informer()
	// use an indexer instead of expensive k8s api calls
	clusterInformer.AddIndexers(map[string]cache.IndexFunc{
		clusterByGenericEngineConfigKey: clusterByKontainerDriver,
	})

	format := Format{
		ClusterIndexer: clusterInformer.GetIndexer(),
	}
	return format.Formatter
}

func CollectionFormatter(apiContext *types.APIContext, collection *types.GenericCollection) {
	collection.AddAction(apiContext, "refresh")
	currContext := apiContext.URLBuilder.Current()
	if !strings.HasSuffix(currContext, "/") {
		currContext = fmt.Sprintf("%s/", currContext)
	}
	collection.Links["rancher-images"] = fmt.Sprintf("%srancher-images", currContext)
}

const clusterByGenericEngineConfigKey = "genericEngineConfig"

// clusterByKontainerDriver is an indexer function that uses the cluster genericEngineConfig
// driverName field
func clusterByKontainerDriver(obj interface{}) ([]string, error) {
	cluster, ok := obj.(*v3.Cluster)
	if !ok {
		return []string{}, nil
	}
	engineConfig := cluster.Spec.GenericEngineConfig
	if engineConfig == nil {
		return []string{}, nil
	}
	driverName, ok := (*engineConfig)["driverName"].(string)
	if !ok {
		return []string{}, nil
	}

	return []string{driverName}, nil
}

func (f *Format) Formatter(request *types.APIContext, resource *types.RawResource) {
	state, ok := resource.Values["state"].(string)
	if ok {
		if state == "active" {
			resource.AddAction(request, "deactivate")
		}

		if state == "inactive" {
			resource.AddAction(request, "activate")
		}
	}
	// if cluster driver is a built-in, delete removal link from UI
	if builtIn, _ := resource.Values[client.KontainerDriverFieldBuiltIn].(bool); builtIn {
		delete(resource.Links, "remove")
		return
	}
	resName := resource.Values["id"]
	// resName will be nil when first added
	if resName != nil {
		clustersWithKontainerDriver, err := f.ClusterIndexer.ByIndex(clusterByGenericEngineConfigKey, resName.(string))
		if err != nil {
			logrus.Warnf("failed to determine if kontainer driver %v was in use by a cluster : %v", resName.(string), err)
		} else if len(clustersWithKontainerDriver) != 0 {
			// if cluster driver in use, delete removal link from UI
			delete(resource.Links, "remove")
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package testutil

import (
	"time"

	federation_api "k8s.io/kubernetes/federation/apis/federation/v1beta1"
	"k8s.io/kubernetes/federation/pkg/federation-controller/util"
	api_v1 "k8s.io/kubernetes/pkg/api/v1"
	"k8s.io/kubernetes/pkg/client/testing/core"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/watch"
)

// RegisterFakeWatch adds a new fake watcher for the specified resource in the given fake client.
// All subsequent requrest for watch on the client will result in returning this fake watcher.
func RegisterFakeWatch(resource string, client *core.Fake) *watch.FakeWatcher {
	watcher := watch.NewFake()
	client.AddWatchReactor(resource, func(action core.Action) (bool, watch.Interface, error) { return true, watcher, nil })
	return watcher
}

// RegisterFakeList registers a list response for the specified resource inside the given fake client.
// The passed value will be returned with every list call.
func RegisterFakeList(resource string, client *core.Fake, obj runtime.Object) {
	client.AddReactor("list", resource, func(action core.Action) (bool, runtime.Object, error) {
		return true, obj, nil
	})
}

// RegisterFakeCopyOnCreate register a reactor in the given fake client that passes
// all created object to the given watcher and also copies them to a channel for
// in-test inspection.
func RegisterFakeCopyOnCreate(resource string, client *core.Fake, watcher *watch.FakeWatcher) chan runtime.Object {
	objChan := make(chan runtime.Object, 100)
	client.AddReactor("create", resource, func(action core.Action) (bool, runtime.Object, error) {
		createAction := action.(core.CreateAction)
		obj := createAction.GetObject()
		go func() {
			watcher.Add(obj)
			objChan <- obj
		}()
		return true, obj, nil
	})
	return objChan
}

// RegisterFakeCopyOnCreate register a reactor in the given fake client that passes
// all updated object to the given watcher and also copies them to a channel for
// in-test inspection.
func RegisterFakeCopyOnUpdate(resource string, client *core.Fake, watcher *watch.FakeWatcher) chan runtime.Object {
	objChan := make(chan runtime.Object, 100)
	client.AddReactor("update", resource, func(action core.Action) (bool, runtime.Object, error) {
		updateAction := action.(core.UpdateAction)
		obj := updateAction.GetObject()
		go func() {
			watcher.Modify(obj)
			objChan <- obj
		}()
		return true, obj, nil
	})
	return objChan
}

// GetObjectFromChan tries to get an api object from the given channel
// within a reasonable time (1 min).
func GetObjectFromChan(c chan runtime.Object) runtime.Object {
	select {
	case obj := <-c:
		return obj
	case <-time.After(time.Minute):
		return nil
	}
}

func ToFederatedInformerForTestOnly(informer util.FederatedInformer) util.FederatedInformerForTestOnly {
	inter := informer.(interface{})
	return inter.(util.FederatedInformerForTestOnly)
}

// NewCluster build a new cluster object.
func NewCluster(name string, readyStatus api_v1.ConditionStatus) *federation_api.Cluster {
	return &federation_api.Cluster{
		ObjectMeta: api_v1.ObjectMeta{
			Name: name,
		},
		Status: federation_api.ClusterStatus{
			Conditions: []federation_api.ClusterCondition{
				{Type: federation_api.ClusterReady, Status: readyStatus},
			},
		},
	}
}
<commit_msg>WatcherDispatcher for federated controller tests<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package testutil

import (
	"os"
	"runtime/pprof"
	"sync"
	"time"

	federation_api "k8s.io/kubernetes/federation/apis/federation/v1beta1"
	"k8s.io/kubernetes/federation/pkg/federation-controller/util"
	api_v1 "k8s.io/kubernetes/pkg/api/v1"
	"k8s.io/kubernetes/pkg/client/testing/core"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/watch"

	"github.com/golang/glog"
)

// A structure that distributes eventes to multiple watchers.
type WatcherDispatcher struct {
	sync.Mutex
	watchers    []*watch.FakeWatcher
	eventsSoFar []*watch.Event
}

func (wd *WatcherDispatcher) register(watcher *watch.FakeWatcher) {
	wd.Lock()
	defer wd.Unlock()
	wd.watchers = append(wd.watchers, watcher)
	for _, event := range wd.eventsSoFar {
		go watcher.Action(event.Type, event.Object)
	}
}

// Add sends an add event.
func (wd *WatcherDispatcher) Add(obj runtime.Object) {
	wd.Lock()
	defer wd.Unlock()
	event := &watch.Event{
		Type:   watch.Added,
		Object: obj,
	}
	wd.eventsSoFar = append(wd.eventsSoFar, event)
	for _, watcher := range wd.watchers {
		go watcher.Add(obj)
	}
}

// Modify sends a modify event.
func (wd *WatcherDispatcher) Modify(obj runtime.Object) {
	wd.Lock()
	defer wd.Unlock()
	event := &watch.Event{
		Type:   watch.Modified,
		Object: obj,
	}
	wd.eventsSoFar = append(wd.eventsSoFar, event)
	for _, watcher := range wd.watchers {
		go watcher.Modify(obj)
	}
}

// Delete sends a delete event.
func (wd *WatcherDispatcher) Delete(lastValue runtime.Object) {
	wd.Lock()
	defer wd.Unlock()
	event := &watch.Event{
		Type:   watch.Deleted,
		Object: lastValue,
	}
	wd.eventsSoFar = append(wd.eventsSoFar, event)
	for _, watcher := range wd.watchers {
		go watcher.Delete(lastValue)
	}
}

// Error sends an Error event.
func (wd *WatcherDispatcher) Error(errValue runtime.Object) {
	wd.Lock()
	defer wd.Unlock()
	event := &watch.Event{
		Type:   watch.Error,
		Object: errValue,
	}
	wd.eventsSoFar = append(wd.eventsSoFar, event)
	for _, watcher := range wd.watchers {
		go watcher.Error(errValue)
	}
}

// Action sends an event of the requested type, for table-based testing.
func (wd *WatcherDispatcher) Action(action watch.EventType, obj runtime.Object) {
	wd.Lock()
	defer wd.Unlock()
	event := &watch.Event{
		Type:   action,
		Object: obj,
	}
	wd.eventsSoFar = append(wd.eventsSoFar, event)
	for _, watcher := range wd.watchers {
		go watcher.Action(action, obj)
	}
}

// RegisterFakeWatch adds a new fake watcher for the specified resource in the given fake client.
// All subsequent requests for a watch on the client will result in returning this fake watcher.
func RegisterFakeWatch(resource string, client *core.Fake) *WatcherDispatcher {
	dispatcher := &WatcherDispatcher{
		watchers:    make([]*watch.FakeWatcher, 0),
		eventsSoFar: make([]*watch.Event, 0),
	}

	client.AddWatchReactor(resource, func(action core.Action) (bool, watch.Interface, error) {
		watcher := watch.NewFake()
		dispatcher.register(watcher)
		return true, watcher, nil
	})
	return dispatcher
}

// RegisterFakeList registers a list response for the specified resource inside the given fake client.
// The passed value will be returned with every list call.
func RegisterFakeList(resource string, client *core.Fake, obj runtime.Object) {
	client.AddReactor("list", resource, func(action core.Action) (bool, runtime.Object, error) {
		return true, obj, nil
	})
}

// RegisterFakeCopyOnCreate registers a reactor in the given fake client that passes
// all created objects to the given watcher and also copies them to a channel for
// in-test inspection.
func RegisterFakeCopyOnCreate(resource string, client *core.Fake, watcher *WatcherDispatcher) chan runtime.Object {
	objChan := make(chan runtime.Object, 100)
	client.AddReactor("create", resource, func(action core.Action) (bool, runtime.Object, error) {
		createAction := action.(core.CreateAction)
		obj := createAction.GetObject()
		go func() {
			watcher.Add(obj)
			objChan <- obj
		}()
		return true, obj, nil
	})
	return objChan
}

// RegisterFakeCopyOnCreate registers a reactor in the given fake client that passes
// all updated objects to the given watcher and also copies them to a channel for
// in-test inspection.
func RegisterFakeCopyOnUpdate(resource string, client *core.Fake, watcher *WatcherDispatcher) chan runtime.Object {
	objChan := make(chan runtime.Object, 100)
	client.AddReactor("update", resource, func(action core.Action) (bool, runtime.Object, error) {
		updateAction := action.(core.UpdateAction)
		obj := updateAction.GetObject()
		go func() {
			glog.V(4).Infof("Object updated. Writing to channel: %v", obj)
			defer func() {
				// Sometimes the channel is already closed.
				if panicVal := recover(); panicVal != nil {
					glog.Errorf("Recovering from panic: %v", panicVal)
				}
			}()
			watcher.Modify(obj)
			objChan <- obj
		}()
		return true, obj, nil
	})
	return objChan
}

// GetObjectFromChan tries to get an api object from the given channel
// within a reasonable time (1 min).
func GetObjectFromChan(c chan runtime.Object) runtime.Object {
	select {
	case obj := <-c:
		return obj
	case <-time.After(10 * time.Second):
		pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
		return nil
	}
}

func ToFederatedInformerForTestOnly(informer util.FederatedInformer) util.FederatedInformerForTestOnly {
	inter := informer.(interface{})
	return inter.(util.FederatedInformerForTestOnly)
}

// NewCluster builds a new cluster object.
func NewCluster(name string, readyStatus api_v1.ConditionStatus) *federation_api.Cluster {
	return &federation_api.Cluster{
		ObjectMeta: api_v1.ObjectMeta{
			Name:        name,
			Annotations: map[string]string{},
		},
		Status: federation_api.ClusterStatus{
			Conditions: []federation_api.ClusterCondition{
				{Type: federation_api.ClusterReady, Status: readyStatus},
			},
		},
	}
}
<|endoftext|>


<commit_before>//
// Copyright (c) 2017 Red Hat, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Red Hat trademarks are not licensed under Apache License, Version 2.
// No permission is granted to use or replicate Red Hat trademarks that
// are incorporated in this software or its documentation.
//

package adapters

import (
	logging "github.com/op/go-logging"
	"github.com/openshift/ansible-service-broker/pkg/apb"
	"github.com/openshift/ansible-service-broker/pkg/clients"
	yaml "gopkg.in/yaml.v1"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"strings"
)

const localOpenShiftName = "openshift-registry"

// LocalOpenShiftAdapter - Docker Hub Adapter
type LocalOpenShiftAdapter struct {
	Config Configuration
	Log    *logging.Logger
}

// RegistryName - Retrieve the registry name
func (r LocalOpenShiftAdapter) RegistryName() string {
	return localOpenShiftName
}

// GetImageNames - retrieve the images
func (r LocalOpenShiftAdapter) GetImageNames() ([]string, error) {
	r.Log.Debug("LocalOpenShiftAdapter::GetImageNames")
	r.Log.Debug("BundleSpecLabel: %s", BundleSpecLabel)

	openshiftClient, err := clients.Openshift(r.Log)
	if err != nil {
		r.Log.Errorf("Failed to instantiate OpenShift client")
		return nil, err
	}

	images, err := openshiftClient.ListRegistryImages(r.Log)
	if err != nil {
		r.Log.Errorf("Failed to load registry images")
		return nil, err
	}

	return images, nil
}

// FetchSpecs - retrieve the spec for the image names.
func (r LocalOpenShiftAdapter) FetchSpecs(imageNames []string) ([]*apb.Spec, error) {
	r.Log.Debug("LocalOpenShiftAdapter::FetchSpecs")
	specList := []*apb.Spec{}
	registryIP, err := r.getServiceIP("docker-registry", "default")
	if err != nil {
		r.Log.Errorf("Failed get docker-registry service information.")
		return nil, err
	}

	openshiftClient, err := clients.Openshift(r.Log)
	if err != nil {
		r.Log.Errorf("Failed to instantiate OpenShift client.")
		return nil, err
	}

	fqImages, err := openshiftClient.ConvertRegistryImagesToSpecs(r.Log, imageNames)
	if err != nil {
		r.Log.Errorf("Failed to load registry images")
		return nil, err
	}

	for _, image := range fqImages {
		spec, err := r.loadSpec(image.DecodedSpec)
		if err != nil {
			r.Log.Errorf("Failed to load image spec")
			continue
		}
		if strings.HasPrefix(image.Name, registryIP) {
			// Image has proper registry IP prefix
			spec.Image = image.Name
			namespace := strings.Split(image.Name, "/")[1]
			for _, ns := range r.Config.Namespaces {
				if ns == namespace {
					r.Log.Debugf("Image [%v] is in configured namespace [%v]. Adding to SpecList.", image.Name, ns)
					specList = append(specList, spec)
				}
			}
		} else {
			r.Log.Debugf("Image does not have proper registry IP prefix. Something went wrong.")
		}
	}

	return specList, nil
}

func (r LocalOpenShiftAdapter) loadSpec(yamlSpec []byte) (*apb.Spec, error) {
	r.Log.Debug("LocalOpenShiftAdapter::LoadSpec")
	spec := &apb.Spec{}

	err := yaml.Unmarshal(yamlSpec, spec)
	if err != nil {
		r.Log.Errorf("Something went wrong loading decoded spec yaml, %s", err)
		return nil, err
	}
	return spec, nil
}

func (r LocalOpenShiftAdapter) getServiceIP(service string, namespace string) (string, error) {
	k8scli, err := clients.Kubernetes(r.Log)
	if err != nil {
		return "", err
	}

	serviceData, err := k8scli.CoreV1().Services(namespace).Get(service, meta_v1.GetOptions{})
	if err != nil {
		r.Log.Warningf("Unable to load service '%s' from namespace '%s'", service, namespace)
		return "", err
	}
	r.Log.Debugf("Found service with name %v", service)

	return serviceData.Spec.ClusterIP, nil
}
<commit_msg>Bug 1507111 - Do not force image tag to be IP + Port (#540)<commit_after>//
// Copyright (c) 2017 Red Hat, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Red Hat trademarks are not licensed under Apache License, Version 2.
// No permission is granted to use or replicate Red Hat trademarks that
// are incorporated in this software or its documentation.
//

package adapters

import (
	logging "github.com/op/go-logging"
	"github.com/openshift/ansible-service-broker/pkg/apb"
	"github.com/openshift/ansible-service-broker/pkg/clients"
	yaml "gopkg.in/yaml.v1"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"strings"
)

const localOpenShiftName = "openshift-registry"

// LocalOpenShiftAdapter - Docker Hub Adapter
type LocalOpenShiftAdapter struct {
	Config Configuration
	Log    *logging.Logger
}

// RegistryName - Retrieve the registry name
func (r LocalOpenShiftAdapter) RegistryName() string {
	return localOpenShiftName
}

// GetImageNames - retrieve the images
func (r LocalOpenShiftAdapter) GetImageNames() ([]string, error) {
	r.Log.Debug("LocalOpenShiftAdapter::GetImageNames")
	r.Log.Debug("BundleSpecLabel: %s", BundleSpecLabel)

	openshiftClient, err := clients.Openshift(r.Log)
	if err != nil {
		r.Log.Errorf("Failed to instantiate OpenShift client")
		return nil, err
	}

	images, err := openshiftClient.ListRegistryImages(r.Log)
	if err != nil {
		r.Log.Errorf("Failed to load registry images")
		return nil, err
	}

	return images, nil
}

// FetchSpecs - retrieve the spec for the image names.
func (r LocalOpenShiftAdapter) FetchSpecs(imageNames []string) ([]*apb.Spec, error) {
	r.Log.Debug("LocalOpenShiftAdapter::FetchSpecs")
	specList := []*apb.Spec{}
	registryIP, err := r.getServiceIP("docker-registry", "default")
	if err != nil {
		r.Log.Errorf("Failed get docker-registry service information.")
		return nil, err
	}

	openshiftClient, err := clients.Openshift(r.Log)
	if err != nil {
		r.Log.Errorf("Failed to instantiate OpenShift client.")
		return nil, err
	}

	fqImages, err := openshiftClient.ConvertRegistryImagesToSpecs(r.Log, imageNames)
	if err != nil {
		r.Log.Errorf("Failed to load registry images")
		return nil, err
	}

	for _, image := range fqImages {
		spec, err := r.loadSpec(image.DecodedSpec)
		if err != nil {
			r.Log.Errorf("Failed to load image spec")
			continue
		}
		if strings.HasPrefix(image.Name, registryIP) == false {
			r.Log.Debugf("Image does not have a registry IP as prefix. This might cause problems but not erroring out.")
		}
		if r.Config.Namespaces == nil {
			r.Log.Debugf("Namespace not set. Assuming `openshift`")
			r.Config.Namespaces = append(r.Config.Namespaces, "openshift")
		}
		spec.Image = image.Name
		nsList := strings.Split(image.Name, "/")
		var namespace string
		if len(nsList) == 0 {
			r.Log.Errorf("Image [%v] is not in the proper format. Erroring.", image.Name)
			continue
		} else if len(nsList) < 3 {
			// Image does not have any registry prefix. May be a product of S2I
			// Expecting openshift/foo-apb
			namespace = nsList[0]
		} else {
			// Expecting format: 172.30.1.1:5000/openshift/foo-apb
			namespace = nsList[1]
		}
		for _, ns := range r.Config.Namespaces {
			if ns == namespace {
				r.Log.Debugf("Image [%v] is in configured namespace [%v]. Adding to SpecList.", image.Name, ns)
				specList = append(specList, spec)
			}
		}
	}

	return specList, nil
}

func (r LocalOpenShiftAdapter) loadSpec(yamlSpec []byte) (*apb.Spec, error) {
	r.Log.Debug("LocalOpenShiftAdapter::LoadSpec")
	spec := &apb.Spec{}

	err := yaml.Unmarshal(yamlSpec, spec)
	if err != nil {
		r.Log.Errorf("Something went wrong loading decoded spec yaml, %s", err)
		return nil, err
	}
	return spec, nil
}

func (r LocalOpenShiftAdapter) getServiceIP(service string, namespace string) (string, error) {
	k8scli, err := clients.Kubernetes(r.Log)
	if err != nil {
		return "", err
	}

	serviceData, err := k8scli.CoreV1().Services(namespace).Get(service, meta_v1.GetOptions{})
	if err != nil {
		r.Log.Warningf("Unable to load service '%s' from namespace '%s'", service, namespace)
		return "", err
	}
	r.Log.Debugf("Found service with name %v", service)

	return serviceData.Spec.ClusterIP, nil
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"reflect"
	"testing"
)

func testIconsFindHelper(terms []string) icons {
	return newIcons().find(terms)
}

func TestIcons_iconsYamlPath_TestEnv(t *testing.T) {
	actual := iconsYamlPath()
	expected := "workflow/icons.yml"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_iconsYamlPath_ProductionEnv(t *testing.T) {
	resetEnv := setTestEnvHelper("FAW_ICONS_YAML_PATH", "")
	defer resetEnv()

	actual := iconsYamlPath()
	expected := "icons.yml"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_iconsReadYaml(t *testing.T) {
	path := "workflow/icons.yml"
	actual, _ := iconsReadYaml(path)

	expected, _ := ioutil.ReadFile(path)
	if !reflect.DeepEqual(actual, expected) {
		t.Error("failed to read file")
	}
}

func TestIcons_iconsReadYaml_Error(t *testing.T) {
	path := ""
	_, err := iconsReadYaml(path)

	if err == nil {
		t.Error("expected error, but nil")
	}
}

func TestIcons_iconsUnmarshalYaml(t *testing.T) {
	b := []byte(`
icons:
- name: Accessible Icon
  id: accessible-icon
  unicode: f368
  created: 5.0.0
  filter:
  - accessibility
  - wheelchair
  - handicap
  - person
  - wheelchair-alt
  categories: unknown
`)
	actual, _ := iconsUnmarshalYaml(b)

	icon := icon{
		Name:    "Accessible Icon",
		ID:      "accessible-icon",
		Unicode: "f368",
		Created: "5.0.0",
		Filter:  []string{"accessibility", "wheelchair", "handicap", "person", "wheelchair-alt"},
	}
	expected := iconsYaml{icons{icon}}
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_AllIcons(t *testing.T) {
	fi := testIconsFindHelper([]string{""})

	actual := len(fi)
	expected := 923
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_ZeroIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{"foo-bar-baz"})

	actual := len(fi)
	expected := 0
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_OneIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{"github-square"})

	actual := len(fi)
	expected := 1
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_TwoIcons(t *testing.T) {
	fi := testIconsFindHelper([]string{"github-"})

	actual := len(fi)
	expected := 2
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_FirstIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{""})

	actual := fi[0].ID
	expected := "500px"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_LastIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{""})

	actual := fi[len(fi)-1].ID
	expected := "youtube-square"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_TaxiIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{"taxi"})

	actual := fi[0].Name
	expected := "Taxi"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].ID
	expected = "taxi"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Unicode
	expected = "f1ba"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Created
	expected = "4.1"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	// actual = fi[0].Aliases[0]
	// expected = "cab"
	// if actual != expected {
	// 	t.Errorf("expected %v to eq %v", actual, expected)
	// }

	actual = fi[0].Filter[0]
	expected = "vehicle"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	// actual = fi[0].Categories[0]
	// expected = "Web Application Icons"
	// if actual != expected {
	// 	t.Errorf("expected %v to eq %v", actual, expected)
	// }
}

// func TestIcons_find_Aliases(t *testing.T) {
// 	fi := testIconsFindHelper([]string{"navicon"})

// 	actual := fi[0].ID
// 	expected := "bars"
// 	if actual != expected {
// 		t.Errorf("expected %v to eq %v", actual, expected)
// 	}

// 	if len(fi) != 1 {
// 		t.Errorf("expected %v to eq %v", len(fi), 1)
// 	}
// }

func TestIcons_findByUnicode(t *testing.T) {
	fi := newIcons().findByUnicode("f067")

	actual := fi[0].ID
	expected := "plus"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	if len(fi) != 1 {
		t.Errorf("expected %v to eq %v", len(fi), 1)
	}
}
<commit_msg>(Font Awesome 5.0.13) Fix test<commit_after>package main

import (
	"io/ioutil"
	"reflect"
	"testing"
)

func testIconsFindHelper(terms []string) icons {
	return newIcons().find(terms)
}

func TestIcons_iconsYamlPath_TestEnv(t *testing.T) {
	actual := iconsYamlPath()
	expected := "workflow/icons.yml"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_iconsYamlPath_ProductionEnv(t *testing.T) {
	resetEnv := setTestEnvHelper("FAW_ICONS_YAML_PATH", "")
	defer resetEnv()

	actual := iconsYamlPath()
	expected := "icons.yml"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_iconsReadYaml(t *testing.T) {
	path := "workflow/icons.yml"
	actual, _ := iconsReadYaml(path)

	expected, _ := ioutil.ReadFile(path)
	if !reflect.DeepEqual(actual, expected) {
		t.Error("failed to read file")
	}
}

func TestIcons_iconsReadYaml_Error(t *testing.T) {
	path := ""
	_, err := iconsReadYaml(path)

	if err == nil {
		t.Error("expected error, but nil")
	}
}

func TestIcons_iconsUnmarshalYaml(t *testing.T) {
	b := []byte(`
icons:
- name: Accessible Icon
  id: accessible-icon
  unicode: f368
  created: 5.0.0
  filter:
  - accessibility
  - wheelchair
  - handicap
  - person
  - wheelchair-alt
  categories: unknown
`)
	actual, _ := iconsUnmarshalYaml(b)

	icon := icon{
		Name:    "Accessible Icon",
		ID:      "accessible-icon",
		Unicode: "f368",
		Created: "5.0.0",
		Filter:  []string{"accessibility", "wheelchair", "handicap", "person", "wheelchair-alt"},
	}
	expected := iconsYaml{icons{icon}}
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_AllIcons(t *testing.T) {
	fi := testIconsFindHelper([]string{""})

	actual := len(fi)
	expected := 989
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_ZeroIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{"foo-bar-baz"})

	actual := len(fi)
	expected := 0
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_OneIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{"github-square"})

	actual := len(fi)
	expected := 1
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_TwoIcons(t *testing.T) {
	fi := testIconsFindHelper([]string{"github-"})

	actual := len(fi)
	expected := 2
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_FirstIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{""})

	actual := fi[0].ID
	expected := "500px"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_LastIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{""})

	actual := fi[len(fi)-1].ID
	expected := "youtube-square"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_TaxiIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{"taxi"})

	actual := fi[0].Name
	expected := "Taxi"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].ID
	expected = "taxi"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Unicode
	expected = "f1ba"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Created
	expected = "4.1"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	// actual = fi[0].Aliases[0]
	// expected = "cab"
	// if actual != expected {
	// 	t.Errorf("expected %v to eq %v", actual, expected)
	// }

	actual = fi[0].Filter[0]
	expected = "vehicle"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	// actual = fi[0].Categories[0]
	// expected = "Web Application Icons"
	// if actual != expected {
	// 	t.Errorf("expected %v to eq %v", actual, expected)
	// }
}

// func TestIcons_find_Aliases(t *testing.T) {
// 	fi := testIconsFindHelper([]string{"navicon"})

// 	actual := fi[0].ID
// 	expected := "bars"
// 	if actual != expected {
// 		t.Errorf("expected %v to eq %v", actual, expected)
// 	}

// 	if len(fi) != 1 {
// 		t.Errorf("expected %v to eq %v", len(fi), 1)
// 	}
// }

func TestIcons_findByUnicode(t *testing.T) {
	fi := newIcons().findByUnicode("f067")

	actual := fi[0].ID
	expected := "plus"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	if len(fi) != 1 {
		t.Errorf("expected %v to eq %v", len(fi), 1)
	}
}
<|endoftext|>


<commit_before>package idea

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/ghthor/journal/git"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
)

// Used to manage idea storage in a directory
type DirectoryStore struct {
	root string
}

// Returned if a directory structure doesn't match
// the required format of an idea storage directory
type InvalidDirectoryStoreError struct {
	Err error
}

func (e InvalidDirectoryStoreError) Error() string {
	return fmt.Sprintf("invalid directory store: %v", e.Err)
}

func IsInvalidDirectoryStoreError(err error) bool {
	_, ok := err.(InvalidDirectoryStoreError)
	return ok
}

func isAnDirectoryStore(d string) error {
	nextIdPath := filepath.Join(d, "nextid")

	data, err := ioutil.ReadFile(nextIdPath)
	if err != nil {
		return InvalidDirectoryStoreError{err}
	}

	var nextAvailableId uint
	n, err := fmt.Fscanf(bytes.NewReader(data), "%d\n", &nextAvailableId)
	if err != nil {
		return InvalidDirectoryStoreError{err}
	}

	if n != 1 {
		return InvalidDirectoryStoreError{errors.New("next available id wasn't found")}
	}

	return nil
}

// Checks that the directory contains the correct files
// to be a DirectoryStore.
// If the directory doesn't contain the require files
// with the expected format this function will return an InvalidDirectoryStoreError.
func NewDirectoryStore(directory string) (*DirectoryStore, error) {
	err := isAnDirectoryStore(directory)
	if err != nil {
		return nil, err
	}

	return &DirectoryStore{directory}, nil
}

// Returned if InitDirectoryStore is called on a directory
// that has already been initialized
var ErrInitOnExistingDirectoryStore = errors.New("init on existing directory store")

type directoryStoreInitialized struct {
	dir     string
	changes []git.CommitableChange
	msg     string
}

func (i directoryStoreInitialized) WorkingDirectory() string {
	return i.dir
}

func (i directoryStoreInitialized) Changes() []git.CommitableChange {
	return i.changes
}

func (i directoryStoreInitialized) CommitMsg() string {
	return i.msg
}

// Check that the directory is empty
// and if it is then it initializes an empty
// idea directory store.
func InitDirectoryStore(directory string) (*DirectoryStore, git.Commitable, error) {
	err := isAnDirectoryStore(directory)
	if err == nil {
		return nil, nil, ErrInitOnExistingDirectoryStore
	}

	nextIdCounter := filepath.Join(directory, "nextid")
	err = ioutil.WriteFile(nextIdCounter, []byte("1\n"), 0600)
	if err != nil {
		return nil, nil, err
	}

	activeIndex := filepath.Join(directory, "active")
	err = ioutil.WriteFile(activeIndex, []byte(""), 0600)
	if err != nil {
		return nil, nil, err
	}

	return &DirectoryStore{directory}, directoryStoreInitialized{
		directory,
		[]git.CommitableChange{
			git.ChangedFile("nextid"),
			git.ChangedFile("active"),
		},
		"directory store initialized",
	}, nil
}

// Saves an idea to the directory store and
// returns a commitable containing all changes.
// If the idea does not have an id it will be assigned one.
// If the idea does have an id it will be updated.
func (d DirectoryStore) SaveIdea(idea *Idea) (git.Commitable, error) {
	return nil, nil
}

var ErrIdeaExists = errors.New("cannot save a new idea because it already exists")

// Saves an idea that doesn't have an id to the directory and
// returns a commitable containing all changes.
// If the idea is already assigned an id this method will
// return ErrIdeaExists
func (d DirectoryStore) SaveNewIdea(idea *Idea) (git.Commitable, error) {
	return d.saveNewIdea(idea)
}

// Does not check if the idea has an id
func (d DirectoryStore) saveNewIdea(idea *Idea) (git.Commitable, error) {
	changes := git.NewChangesIn(d.root)

	// Retrieve nextid
	data, err := ioutil.ReadFile(filepath.Join(d.root, "nextid"))
	if err != nil {
		return nil, err
	}

	var nextId uint
	_, err = fmt.Fscan(bytes.NewReader(data), &nextId)
	if err != nil {
		return nil, err
	}

	idea.Id = nextId

	// Increment nextid
	nextId++

	err = ioutil.WriteFile(filepath.Join(d.root, "nextid"), []byte(fmt.Sprintf("%d\n", nextId)), 0600)
	if err != nil {
		return nil, err
	}
	changes.Add(git.ChangedFile("nextid"))

	// write to file
	r, err := NewIdeaReader(*idea)
	if err != nil {
		return nil, err
	}

	ideaFile, err := os.OpenFile(filepath.Join(d.root, fmt.Sprint(idea.Id)), os.O_CREATE|os.O_WRONLY, 0600)
	if err != nil {
		return nil, err
	}
	defer ideaFile.Close()

	_, err = io.Copy(ideaFile, r)
	if err != nil {
		return nil, err
	}
	changes.Add(git.ChangedFile(filepath.Base(ideaFile.Name())))

	// If Active, append to active index
	if idea.Status == IS_Active {
		activeIndexFile, err := os.OpenFile(filepath.Join(d.root, "active"), os.O_APPEND|os.O_WRONLY, 0600)
		if err != nil {
			return nil, err
		}
		defer activeIndexFile.Close()

		_, err = fmt.Fprintln(activeIndexFile, idea.Id)
		if err != nil {
			return nil, err
		}
		changes.Add(git.ChangedFile("active"))
	}

	changes.Msg = fmt.Sprintf("IDEA - %d - Created", idea.Id)

	return changes, nil
}

var ErrIdeaNotModified = errors.New("the idea was not modified")

// Updates an idea that has already been assigned an id and
// exists in the directory already and
// returns a commitable containing all changes.
// If the idea body wasn't modified this method will
// return ErrIdeaNotModified
func (d DirectoryStore) UpdateIdea(idea Idea) (git.Commitable, error) {
	return nil, nil
}
<commit_msg>Utilize the convenience type provided by the git package<commit_after>package idea

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/ghthor/journal/git"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
)

// Used to manage idea storage in a directory
type DirectoryStore struct {
	root string
}

// Returned if a directory structure doesn't match
// the required format of an idea storage directory
type InvalidDirectoryStoreError struct {
	Err error
}

func (e InvalidDirectoryStoreError) Error() string {
	return fmt.Sprintf("invalid directory store: %v", e.Err)
}

func IsInvalidDirectoryStoreError(err error) bool {
	_, ok := err.(InvalidDirectoryStoreError)
	return ok
}

func isAnDirectoryStore(d string) error {
	nextIdPath := filepath.Join(d, "nextid")

	data, err := ioutil.ReadFile(nextIdPath)
	if err != nil {
		return InvalidDirectoryStoreError{err}
	}

	var nextAvailableId uint
	n, err := fmt.Fscanf(bytes.NewReader(data), "%d\n", &nextAvailableId)
	if err != nil {
		return InvalidDirectoryStoreError{err}
	}

	if n != 1 {
		return InvalidDirectoryStoreError{errors.New("next available id wasn't found")}
	}

	return nil
}

// Checks that the directory contains the correct files
// to be a DirectoryStore.
// If the directory doesn't contain the require files
// with the expected format this function will return an InvalidDirectoryStoreError.
func NewDirectoryStore(directory string) (*DirectoryStore, error) {
	err := isAnDirectoryStore(directory)
	if err != nil {
		return nil, err
	}

	return &DirectoryStore{directory}, nil
}

// Returned if InitDirectoryStore is called on a directory
// that has already been initialized
var ErrInitOnExistingDirectoryStore = errors.New("init on existing directory store")

// Check that the directory is empty
// and if it is then it initializes an empty
// idea directory store.
func InitDirectoryStore(directory string) (*DirectoryStore, git.Commitable, error) {
	err := isAnDirectoryStore(directory)
	if err == nil {
		return nil, nil, ErrInitOnExistingDirectoryStore
	}

	nextIdCounter := filepath.Join(directory, "nextid")
	err = ioutil.WriteFile(nextIdCounter, []byte("1\n"), 0600)
	if err != nil {
		return nil, nil, err
	}

	activeIndex := filepath.Join(directory, "active")
	err = ioutil.WriteFile(activeIndex, []byte(""), 0600)
	if err != nil {
		return nil, nil, err
	}

	changes := git.NewChangesIn(directory)
	changes.Add(git.ChangedFile("nextid"))
	changes.Add(git.ChangedFile("active"))
	changes.Msg = "directory store initialized"

	return &DirectoryStore{directory}, changes, nil
}

// Saves an idea to the directory store and
// returns a commitable containing all changes.
// If the idea does not have an id it will be assigned one.
// If the idea does have an id it will be updated.
func (d DirectoryStore) SaveIdea(idea *Idea) (git.Commitable, error) {
	return nil, nil
}

var ErrIdeaExists = errors.New("cannot save a new idea because it already exists")

// Saves an idea that doesn't have an id to the directory and
// returns a commitable containing all changes.
// If the idea is already assigned an id this method will
// return ErrIdeaExists
func (d DirectoryStore) SaveNewIdea(idea *Idea) (git.Commitable, error) {
	return d.saveNewIdea(idea)
}

// Does not check if the idea has an id
func (d DirectoryStore) saveNewIdea(idea *Idea) (git.Commitable, error) {
	changes := git.NewChangesIn(d.root)

	// Retrieve nextid
	data, err := ioutil.ReadFile(filepath.Join(d.root, "nextid"))
	if err != nil {
		return nil, err
	}

	var nextId uint
	_, err = fmt.Fscan(bytes.NewReader(data), &nextId)
	if err != nil {
		return nil, err
	}

	idea.Id = nextId

	// Increment nextid
	nextId++

	err = ioutil.WriteFile(filepath.Join(d.root, "nextid"), []byte(fmt.Sprintf("%d\n", nextId)), 0600)
	if err != nil {
		return nil, err
	}
	changes.Add(git.ChangedFile("nextid"))

	// write to file
	r, err := NewIdeaReader(*idea)
	if err != nil {
		return nil, err
	}

	ideaFile, err := os.OpenFile(filepath.Join(d.root, fmt.Sprint(idea.Id)), os.O_CREATE|os.O_WRONLY, 0600)
	if err != nil {
		return nil, err
	}
	defer ideaFile.Close()

	_, err = io.Copy(ideaFile, r)
	if err != nil {
		return nil, err
	}
	changes.Add(git.ChangedFile(filepath.Base(ideaFile.Name())))

	// If Active, append to active index
	if idea.Status == IS_Active {
		activeIndexFile, err := os.OpenFile(filepath.Join(d.root, "active"), os.O_APPEND|os.O_WRONLY, 0600)
		if err != nil {
			return nil, err
		}
		defer activeIndexFile.Close()

		_, err = fmt.Fprintln(activeIndexFile, idea.Id)
		if err != nil {
			return nil, err
		}
		changes.Add(git.ChangedFile("active"))
	}

	changes.Msg = fmt.Sprintf("IDEA - %d - Created", idea.Id)

	return changes, nil
}

var ErrIdeaNotModified = errors.New("the idea was not modified")

// Updates an idea that has already been assigned an id and
// exists in the directory already and
// returns a commitable containing all changes.
// If the idea body wasn't modified this method will
// return ErrIdeaNotModified
func (d DirectoryStore) UpdateIdea(idea Idea) (git.Commitable, error) {
	return nil, nil
}
<|endoftext|>


<commit_before>package tool

import (
	"bytes"
	"errors"
	"fmt"
	"os"
	"strings"

	"github.com/voidint/gbb/config"
	"github.com/voidint/gbb/variable"
)

// Builder 编译工具
type Builder interface {
	Build(dir string) error
}

var (
	// ErrBuildTool 不支持的编译工具错误
	ErrBuildTool = errors.New("unsupported build tool")
)

// Build 根据配置信息，调用合适的编译工具进行编译。
// 若配置的编译工具不在支持的工具范围内，则返回ErrBuildTool错误。
func Build(conf *config.Config, dir string) (err error) {
	defer chdir(dir, conf.Debug) // init work directory

	if strings.HasPrefix(conf.Tool, "go ") {
		return NewGoBuilder(conf).Build(dir)
	} else if strings.HasPrefix(conf.Tool, "gb ") {
		return NewGBBuilder(conf).Build(dir)
	}
	return ErrBuildTool
}

func chdir(dir string, debug bool) (err error) {
	wd, err := os.Getwd()
	if err != nil {
		return err
	}
	if wd == dir {
		return nil
	}

	if debug {
		fmt.Printf("==> cd %s\n", dir)
	}
	return os.Chdir(dir)
}

func ldflags(conf *config.Config) (flags string, err error) {
	var buf bytes.Buffer

	if val := Args(strings.Fields(conf.Tool)).ExtractLdflags(); val != "" {
		buf.WriteString(val)
		buf.WriteByte(' ')
	}

	for i := range conf.Variables {
		varName := strings.TrimSpace(conf.Variables[i].Variable)
		varExpr := strings.TrimSpace(conf.Variables[i].Value)

		if conf.Debug {
			fmt.Printf("==> eval(%q)\n", varExpr)
		}
		val, err := variable.Eval(varExpr, conf.Debug)
		if err != nil {
			return "", err
		}
		if conf.Debug {
			fmt.Println(val)
		}
		buf.WriteString(fmt.Sprintf(`-X "%s.%s=%s"`, conf.Importpath, varName, val))
		if i < len(conf.Variables)-1 {
			buf.WriteByte(' ')
		}
	}
	return buf.String(), nil
}

// Args 命令行参数
type Args []string

// ExtractLdflags 抽取参数中ldflags所对应的值
func (args Args) ExtractLdflags() string {
	f := func(r rune) bool {
		return r == rune('"') || r == rune('\'')
	}
	for i, arg := range args {
		if !strings.Contains(arg, "-ldflags") {
			continue
		}
		// eg. go build -ldflags='-w'
		idx := strings.Index(arg, "-ldflags=")
		if idx > -1 {
			return strings.TrimFunc(arg[idx+len("-ldflags="):], f)
		}
		if i >= len(args)-1 || !strings.HasSuffix(arg, "-ldflags") {
			return ""
		}
		// eg. go build -ldflags "-w"
		return strings.TrimFunc(args[i+1], f)
	}
	return ""
}

// RemoveLdflags 移除ldflags参数及其值
func (args Args) RemoveLdflags() (news Args) {
	for i := range args {
		if !strings.Contains(args[i], "-ldflags") {
			continue
		}
		// eg. go build -ldflags='-w'
		if strings.Contains(args[i], "-ldflags=") {
			args[i] = ""
			continue
		}
		// eg. go build -ldflags "-w"
		if i < len(args)-1 && strings.HasSuffix(args[i], "-ldflags") {
			args[i] = ""
			args[i+1] = ""
			continue
		}
	}

	for i := range args {
		if args[i] == "" {
			continue
		}
		news = append(news, args[i])
	}
	return news
}
<commit_msg>Fixbug: The result of extracting -ldflags option value is incorrect<commit_after>package tool

import (
	"bytes"
	"errors"
	"fmt"
	"os"
	"strings"

	"github.com/voidint/gbb/config"
	"github.com/voidint/gbb/variable"
)

// Builder 编译工具
type Builder interface {
	Build(dir string) error
}

var (
	// ErrBuildTool 不支持的编译工具错误
	ErrBuildTool = errors.New("unsupported build tool")
)

// Build 根据配置信息，调用合适的编译工具进行编译。
// 若配置的编译工具不在支持的工具范围内，则返回ErrBuildTool错误。
func Build(conf *config.Config, dir string) (err error) {
	defer chdir(dir, conf.Debug) // init work directory

	if strings.HasPrefix(conf.Tool, "go ") {
		return NewGoBuilder(conf).Build(dir)
	} else if strings.HasPrefix(conf.Tool, "gb ") {
		return NewGBBuilder(conf).Build(dir)
	}
	return ErrBuildTool
}

func chdir(dir string, debug bool) (err error) {
	wd, err := os.Getwd()
	if err != nil {
		return err
	}
	if wd == dir {
		return nil
	}

	if debug {
		fmt.Printf("==> cd %s\n", dir)
	}
	return os.Chdir(dir)
}

func ldflags(conf *config.Config) (flags string, err error) {
	var buf bytes.Buffer

	if val := Args(strings.Fields(conf.Tool)).ExtractLdflags(); val != "" {
		buf.WriteString(val)
		buf.WriteByte(' ')
	}

	for i := range conf.Variables {
		varName := strings.TrimSpace(conf.Variables[i].Variable)
		varExpr := strings.TrimSpace(conf.Variables[i].Value)

		if conf.Debug {
			fmt.Printf("==> eval(%q)\n", varExpr)
		}
		val, err := variable.Eval(varExpr, conf.Debug)
		if err != nil {
			return "", err
		}
		if conf.Debug {
			fmt.Println(val)
		}
		buf.WriteString(fmt.Sprintf(`-X "%s.%s=%s"`, conf.Importpath, varName, val))
		if i < len(conf.Variables)-1 {
			buf.WriteByte(' ')
		}
	}
	return buf.String(), nil
}

// Args 命令行参数
type Args []string

// ExtractLdflags 抽取参数中ldflags所对应的值
func (args Args) ExtractLdflags() string {
	for i, arg := range args {
		if !strings.Contains(arg, "-ldflags") {
			continue
		}
		// eg. go build -ldflags='-w'
		idx := strings.Index(arg, "-ldflags=")
		if idx > -1 {
			return TrimQuotationMarks(arg[idx+len("-ldflags="):])
		}
		if i >= len(args)-1 || !strings.HasSuffix(arg, "-ldflags") {
			return ""
		}
		// eg. go build -ldflags "-w"
		return TrimQuotationMarks(args[i+1])
	}
	return ""
}

// RemoveLdflags 移除ldflags参数及其值
func (args Args) RemoveLdflags() (news Args) {
	for i := range args {
		if !strings.Contains(args[i], "-ldflags") {
			continue
		}
		// eg. go build -ldflags='-w'
		if strings.Contains(args[i], "-ldflags=") {
			args[i] = ""
			continue
		}
		// eg. go build -ldflags "-w"
		if i < len(args)-1 && strings.HasSuffix(args[i], "-ldflags") {
			args[i] = ""
			args[i+1] = ""
			continue
		}
	}

	for i := range args {
		if args[i] == "" {
			continue
		}
		news = append(news, args[i])
	}
	return news
}

// TrimQuotationMarks 去除字符串前后的单/双引号
func TrimQuotationMarks(val string) string {
	if strings.HasSuffix(val, `'`) {
		return strings.TrimPrefix(strings.TrimSuffix(val, `'`), `'`)
	} else if strings.HasSuffix(val, `"`) {
		return strings.TrimPrefix(strings.TrimSuffix(val, `"`), `"`)
	}
	return val
}
<|endoftext|>


<commit_before>package git

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"log"
	"os/exec"
	"regexp"
)

const (
	add = `A`
	mod = `M`
	del = `D`
)

var (
	re = regexp.MustCompile(`^([AMD])\s+(.*)`)
)

// Diff `git diff --name-status HEAD~ HEAD`
func Diff(repo string) (adds, mods, dels []string, err error) {
	script := fmt.Sprintf("cd %s && git diff --name-status HEAD~ HEAD", repo)
	log.Printf("run shell script: %q", script)
	cmd := exec.Command("/bin/sh", "-c", script)
	b, err := cmd.Output()
	if err != nil {
		return
	}
	adds, mods, dels = diff(bytes.NewReader(b))
	return
}

func diff(in io.Reader) (adds, mods, dels []string) {
	scanner := bufio.NewScanner(in)
	for scanner.Scan() {
		line := scanner.Text()
		pair := re.FindStringSubmatch(line)
		if len(pair) != 3 {
			continue
		}
		switch pair[1] {
		case add:
			adds = append(adds, pair[2])
		case mod:
			mods = append(mods, pair[2])
		case del:
			dels = append(dels, pair[2])
		default:
			log.Printf("Unknown git diff line: %q", line)
		}
	}
	if err := scanner.Err(); err != nil {
		log.Printf("parsing git diff: %v", err)
	}
	return
}
<commit_msg>Add Pull cmd in git package.<commit_after>package git

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"log"
	"os/exec"
	"regexp"
)

const (
	add = `A`
	mod = `M`
	del = `D`
)

var (
	re = regexp.MustCompile(`^([AMD])\s+(.*)`)
)

// Diff `git diff --name-status HEAD~ HEAD`
func Diff(repo string) (adds, mods, dels []string, err error) {
	script := fmt.Sprintf("cd %s && git diff --name-status HEAD~ HEAD", repo)
	log.Printf("exec script:\n %q", script)
	cmd := exec.Command("/bin/sh", "-c", script)
	b, err := cmd.Output()
	if err != nil {
		return
	}
	adds, mods, dels = diff(bytes.NewReader(b))
	return
}

// Pull `git pull`
func Pull(repo string) error {
	script := fmt.Sprintf("cd %s && git pull;", repo)
	log.Printf("exec sciprt:\n %q", script)
	cmd := exec.Command("/bin/sh", "-c", script)
	b, err := cmd.Output()
	if err != nil {
		return err
	}
	// print output
	fmt.Printf("%s\n", b)
	return nil
}

func diff(in io.Reader) (adds, mods, dels []string) {
	scanner := bufio.NewScanner(in)
	for scanner.Scan() {
		line := scanner.Text()
		pair := re.FindStringSubmatch(line)
		if len(pair) != 3 {
			continue
		}
		switch pair[1] {
		case add:
			adds = append(adds, pair[2])
		case mod:
			mods = append(mods, pair[2])
		case del:
			dels = append(dels, pair[2])
		default:
			log.Printf("Unknown git diff line: %q", line)
		}
	}
	if err := scanner.Err(); err != nil {
		log.Printf("parsing git diff: %v", err)
	}
	return
}
<|endoftext|>


<commit_before>package git

import (
	"log"
	"os"
	"os/exec"
)

func Init(path string) {
	if err := os.MkdirAll(path, os.ModeDir); err != nil {
		log.Fatal(err)
	}
	err := exec.Command("git", "init", path).Run()
	if err != nil {
		log.Fatal(err)
	}
}

func Add(path, filename string) {
	os.Chdir(path)
	err := exec.Command("git", "add", filename).Run()
	if err != nil {
		log.Fatal(err)
	}
}

func Commit(path, message, date string) {
	os.Chdir(path)
	err := exec.Command("git", "commit", "-m", message, "--date", date).Run()
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Add docs for git.<commit_after>package git

import (
	"log"
	"os"
	"os/exec"
)

// Init initializes a git repo in the given path.
// If the path does not already exists, it will be created.
func Init(path string) {
	if err := os.MkdirAll(path, os.ModeDir); err != nil {
		log.Fatal(err)
	}
	err := exec.Command("git", "init", path).Run()
	if err != nil {
		log.Fatal(err)
	}
}

// Add wraps the git add call and will change into the path of the git repo
// and add the file given by name.
func Add(path, filename string) {
	os.Chdir(path)
	err := exec.Command("git", "add", filename).Run()
	if err != nil {
		log.Fatal(err)
	}
}

// Commit will change into the path of the git repo and execute git commit.
// In addition a message and date for the commit are specified.
func Commit(path, message, date string) {
	os.Chdir(path)
	err := exec.Command("git", "commit", "-m", message, "--date", date).Run()
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package gitmock

//go:generate go run _tools/gen.go
import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/Masterminds/semver"
	"github.com/pkg/errors"
)

// New returns new GitMock
func New(opts ...string) (*GitMock, error) {
	git := "git"
	if len(opts) > 0 {
		git = opts[0]
	}

	cmd := exec.Command(git, "version")
	cmd.Env = append(os.Environ(), "LANG=C")
	var b bytes.Buffer
	cmd.Stdout = &b
	err := cmd.Run()
	if err != nil {
		return nil, errors.Wrap(err, "failed to create new GitMock")
	}
	gitVer := b.String()
	arr := strings.Fields(gitVer)
	if len(arr) != 3 || arr[0] != "git" || arr[1] != "version" {
		return nil, fmt.Errorf("output of `git version` looks strange: %s", gitVer)
	}
	verArr := strings.Split(arr[2], ".")
	if len(verArr) < 3 {
		return nil, fmt.Errorf("git version [%s] looks strange", arr[2])
	}
	semv := strings.Join(verArr[0:2], ".")
	ver, err := semver.NewVersion(semv)
	if err != nil {
		return nil, errors.Wrap(err, fmt.Sprintf("git version [%s] looks strange", arr[2]))
	}
	c, _ := semver.NewConstraint(">= 1.8.5")
	if !c.Check(ver) {
		return nil, fmt.Errorf("git 1.8.5 or later required.")
	}

	user := ""
	err = exec.Command(git, "config", "user.name").Run()
	if err != nil {
		user = "gomock"
	}
	email := ""
	err = exec.Command(git, "config", "user.email").Run()
	if err != nil {
		email = "gomock@example.com"
	}

	return &GitMock{
		gitPath: git,
		user:    user,
		email:   email,
	}, nil
}

// GitMock is git mock repository
type GitMock struct {
	repoPath string
	gitPath  string
	user     string
	email    string
}

// RepoPath returns repository path
func (gm *GitMock) RepoPath() string {
	if gm.repoPath == "" {
		dir, err := ioutil.TempDir("", "gitmock")
		if err != nil {
			log.Fatal(err.Error())
		}
		gm.repoPath = dir
	}
	return gm.repoPath
}

func (gm *GitMock) gitProg() string {
	if gm.gitPath != "" {
		return gm.gitPath
	}
	return "git"
}

func (gm *GitMock) env() (ret []string) {
	if gm.user != "" {
		envs := []string{"GIT_AUTHOR_NAME", "GIT_COMMITER_NAME"}
		for _, v := range envs {
			if env := os.Getenv(v); env == "" {
				ret = append(ret, fmt.Sprintf("%s=%s", v, gm.user))
			}
		}
	}
	if gm.email != "" {
		envs := []string{"GIT_AUTHOR_EMAIL", "GIT_COMMITER_EMAIL"}
		for _, v := range envs {
			if env := os.Getenv(v); env == "" {
				ret = append(ret, fmt.Sprintf("%s=%s", v, gm.email))
			}
		}
	}
	return ret
}

// Do the git command
func (gm *GitMock) Do(args ...string) (string, string, error) {
	arg := []string{"-C", gm.RepoPath()}
	arg = append(arg, args...)
	cmd := exec.Command(gm.gitProg(), arg...)
	cmd.Env = append(os.Environ(), "LANG=C")
	env := gm.env()
	if len(env) > 0 {
		cmd.Env = append(cmd.Env, env...)
	}
	var bout, berr bytes.Buffer
	cmd.Stdout = &bout
	cmd.Stderr = &berr
	err := cmd.Run()
	return bout.String(), berr.String(), err
}

// PutFile put a file to repo
func (gm *GitMock) PutFile(file, content string) error {
	repo := gm.RepoPath()
	fpath := filepath.Join(repo, file)
	err := os.MkdirAll(filepath.Dir(fpath), 0755)
	if err != nil {
		return err
	}
	c := []byte(content)
	return ioutil.WriteFile(fpath, c, 0644)
}
<commit_msg>fix slice<commit_after>package gitmock

//go:generate go run _tools/gen.go
import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/Masterminds/semver"
	"github.com/pkg/errors"
)

// New returns new GitMock
func New(opts ...string) (*GitMock, error) {
	git := "git"
	if len(opts) > 0 {
		git = opts[0]
	}

	cmd := exec.Command(git, "version")
	cmd.Env = append(os.Environ(), "LANG=C")
	var b bytes.Buffer
	cmd.Stdout = &b
	err := cmd.Run()
	if err != nil {
		return nil, errors.Wrap(err, "failed to create new GitMock")
	}
	gitVer := b.String()
	arr := strings.Fields(gitVer)
	if len(arr) != 3 || arr[0] != "git" || arr[1] != "version" {
		return nil, fmt.Errorf("output of `git version` looks strange: %s", gitVer)
	}
	verArr := strings.Split(arr[2], ".")
	if len(verArr) < 3 {
		return nil, fmt.Errorf("git version [%s] looks strange", arr[2])
	}
	semv := strings.Join(verArr[:3], ".")
	ver, err := semver.NewVersion(semv)
	if err != nil {
		return nil, errors.Wrap(err, fmt.Sprintf("git version [%s] looks strange", arr[2]))
	}
	c, _ := semver.NewConstraint(">= 1.8.5")
	if !c.Check(ver) {
		return nil, fmt.Errorf("git 1.8.5 or later required")
	}

	user := ""
	err = exec.Command(git, "config", "user.name").Run()
	if err != nil {
		user = "gomock"
	}
	email := ""
	err = exec.Command(git, "config", "user.email").Run()
	if err != nil {
		email = "gomock@example.com"
	}

	return &GitMock{
		gitPath: git,
		user:    user,
		email:   email,
	}, nil
}

// GitMock is git mock repository
type GitMock struct {
	repoPath string
	gitPath  string
	user     string
	email    string
}

// RepoPath returns repository path
func (gm *GitMock) RepoPath() string {
	if gm.repoPath == "" {
		dir, err := ioutil.TempDir("", "gitmock")
		if err != nil {
			log.Fatal(err.Error())
		}
		gm.repoPath = dir
	}
	return gm.repoPath
}

func (gm *GitMock) gitProg() string {
	if gm.gitPath != "" {
		return gm.gitPath
	}
	return "git"
}

func (gm *GitMock) env() (ret []string) {
	if gm.user != "" {
		envs := []string{"GIT_AUTHOR_NAME", "GIT_COMMITER_NAME"}
		for _, v := range envs {
			if env := os.Getenv(v); env == "" {
				ret = append(ret, fmt.Sprintf("%s=%s", v, gm.user))
			}
		}
	}
	if gm.email != "" {
		envs := []string{"GIT_AUTHOR_EMAIL", "GIT_COMMITER_EMAIL"}
		for _, v := range envs {
			if env := os.Getenv(v); env == "" {
				ret = append(ret, fmt.Sprintf("%s=%s", v, gm.email))
			}
		}
	}
	return ret
}

// Do the git command
func (gm *GitMock) Do(args ...string) (string, string, error) {
	arg := []string{"-C", gm.RepoPath()}
	arg = append(arg, args...)
	cmd := exec.Command(gm.gitProg(), arg...)
	cmd.Env = append(os.Environ(), "LANG=C")
	env := gm.env()
	if len(env) > 0 {
		cmd.Env = append(cmd.Env, env...)
	}
	var bout, berr bytes.Buffer
	cmd.Stdout = &bout
	cmd.Stderr = &berr
	err := cmd.Run()
	return bout.String(), berr.String(), err
}

// PutFile put a file to repo
func (gm *GitMock) PutFile(file, content string) error {
	repo := gm.RepoPath()
	fpath := filepath.Join(repo, file)
	err := os.MkdirAll(filepath.Dir(fpath), 0755)
	if err != nil {
		return err
	}
	c := []byte(content)
	return ioutil.WriteFile(fpath, c, 0644)
}
<|endoftext|>


<commit_before>/*
* @Author: Xier
* @Date:   2015-02-02 12:35:19
* @Last Modified by:   Xier
* @Last Modified time: 2015-02-02 16:04:12
 */

package gika

import (
	"bytes"
	"github.com/stretchr/testify/assert"
	"os"
	"testing"
)

func TestReadDocument(t *testing.T) {
	// Create io.Reader
	f, err := os.Open("./doc-example/file.pdf")
	defer func() {
		f.Close()
	}()
	if err != nil {
		panic(err)
	}

	// Create io.Writer
	ws := &bytes.Buffer{}

	err = ReadDocument(f, ws)
	assert.NoError(t, err)
	assert.True(t, ws.Len() > 0)
}

func TestIsSupport(t *testing.T) {
	sup := IsSupport("./example/file.doc")
	assert.True(t, sup)
}

func TestIsSupportFalse(t *testing.T) {
	sup := IsSupport("./example/file.docc")
	assert.False(t, sup)
}

func TestIsSupportNoPath(t *testing.T) {
	sup := IsSupport("file.docx")
	assert.True(t, sup)
}
<commit_msg>fix test change method name<commit_after>/*
* @Author: Xier
* @Date:   2015-02-02 12:35:19
* @Last Modified by:   Xier
* @Last Modified time: 2015-02-02 16:22:50
 */

package gika

import (
	"bytes"
	"github.com/stretchr/testify/assert"
	"os"
	"testing"
)

func TestReadDocument(t *testing.T) {
	// Create io.Reader
	f, err := os.Open("./doc-example/file.pdf")
	defer func() {
		f.Close()
	}()
	if err != nil {
		panic(err)
	}

	// Create io.Writer
	ws := &bytes.Buffer{}

	err = DocToText(f, ws)
	assert.NoError(t, err)
	assert.True(t, ws.Len() > 0)
}

func TestIsSupport(t *testing.T) {
	sup := IsSupport("./example/file.doc")
	assert.True(t, sup)
}

func TestIsSupportFalse(t *testing.T) {
	sup := IsSupport("./example/file.docc")
	assert.False(t, sup)
}

func TestIsSupportNoPath(t *testing.T) {
	sup := IsSupport("file.docx")
	assert.True(t, sup)
}
<|endoftext|>


<commit_before>// package bitswap implements the IPFS Exchange interface with the BitSwap
// bilateral exchange protocol.
package bitswap

import (
	"errors"
	"math"
	"sync"
	"time"

	process "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess"
	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
	blocks "github.com/ipfs/go-ipfs/blocks"
	blockstore "github.com/ipfs/go-ipfs/blocks/blockstore"
	exchange "github.com/ipfs/go-ipfs/exchange"
	decision "github.com/ipfs/go-ipfs/exchange/bitswap/decision"
	bsmsg "github.com/ipfs/go-ipfs/exchange/bitswap/message"
	bsnet "github.com/ipfs/go-ipfs/exchange/bitswap/network"
	notifications "github.com/ipfs/go-ipfs/exchange/bitswap/notifications"
	wantlist "github.com/ipfs/go-ipfs/exchange/bitswap/wantlist"
	peer "github.com/ipfs/go-ipfs/p2p/peer"
	"github.com/ipfs/go-ipfs/thirdparty/delay"
	eventlog "github.com/ipfs/go-ipfs/thirdparty/eventlog"
	u "github.com/ipfs/go-ipfs/util"
)

var log = eventlog.Logger("bitswap")

const (
	// maxProvidersPerRequest specifies the maximum number of providers desired
	// from the network. This value is specified because the network streams
	// results.
	// TODO: if a 'non-nice' strategy is implemented, consider increasing this value
	maxProvidersPerRequest = 3
	providerRequestTimeout = time.Second * 10
	hasBlockTimeout        = time.Second * 15
	provideTimeout         = time.Second * 15
	sizeBatchRequestChan   = 32
	// kMaxPriority is the max priority as defined by the bitswap protocol
	kMaxPriority = math.MaxInt32

	HasBlockBufferSize = 256
	provideWorkers     = 4
)

var rebroadcastDelay = delay.Fixed(time.Second * 10)

// New initializes a BitSwap instance that communicates over the provided
// BitSwapNetwork. This function registers the returned instance as the network
// delegate.
// Runs until context is cancelled.
func New(parent context.Context, p peer.ID, network bsnet.BitSwapNetwork,
	bstore blockstore.Blockstore, nice bool) exchange.Interface {

	// important to use provided parent context (since it may include important
	// loggable data). It's probably not a good idea to allow bitswap to be
	// coupled to the concerns of the IPFS daemon in this way.
	//
	// FIXME(btc) Now that bitswap manages itself using a process, it probably
	// shouldn't accept a context anymore. Clients should probably use Close()
	// exclusively. We should probably find another way to share logging data
	ctx, cancelFunc := context.WithCancel(parent)

	notif := notifications.New()
	px := process.WithTeardown(func() error {
		notif.Shutdown()
		return nil
	})

	go func() {
		<-px.Closing() // process closes first
		cancelFunc()
	}()
	go func() {
		<-ctx.Done() // parent cancelled first
		px.Close()
	}()

	bs := &Bitswap{
		self:          p,
		blockstore:    bstore,
		notifications: notif,
		engine:        decision.NewEngine(ctx, bstore), // TODO close the engine with Close() method
		network:       network,
		findKeys:      make(chan *blockRequest, sizeBatchRequestChan),
		process:       px,
		newBlocks:     make(chan *blocks.Block, HasBlockBufferSize),
		provideKeys:   make(chan u.Key),
		wm:            NewWantManager(ctx, network),
	}
	go bs.wm.Run()
	network.SetDelegate(bs)

	// Start up bitswaps async worker routines
	bs.startWorkers(px, ctx)
	return bs
}

// Bitswap instances implement the bitswap protocol.
type Bitswap struct {

	// the ID of the peer to act on behalf of
	self peer.ID

	// network delivers messages on behalf of the session
	network bsnet.BitSwapNetwork

	// the peermanager manages sending messages to peers in a way that
	// wont block bitswap operation
	wm *WantManager

	// blockstore is the local database
	// NB: ensure threadsafety
	blockstore blockstore.Blockstore

	notifications notifications.PubSub

	// send keys to a worker to find and connect to providers for them
	findKeys chan *blockRequest

	engine *decision.Engine

	process process.Process

	newBlocks chan *blocks.Block

	provideKeys chan u.Key

	counterLk      sync.Mutex
	blocksRecvd    int
	dupBlocksRecvd int
}

type blockRequest struct {
	keys []u.Key
	ctx  context.Context
}

// GetBlock attempts to retrieve a particular block from peers within the
// deadline enforced by the context.
func (bs *Bitswap) GetBlock(parent context.Context, k u.Key) (*blocks.Block, error) {

	// Any async work initiated by this function must end when this function
	// returns. To ensure this, derive a new context. Note that it is okay to
	// listen on parent in this scope, but NOT okay to pass |parent| to
	// functions called by this one. Otherwise those functions won't return
	// when this context's cancel func is executed. This is difficult to
	// enforce. May this comment keep you safe.

	ctx, cancelFunc := context.WithCancel(parent)

	ctx = eventlog.ContextWithLoggable(ctx, eventlog.Uuid("GetBlockRequest"))
	defer log.EventBegin(ctx, "GetBlockRequest", &k).Done()

	defer func() {
		cancelFunc()
	}()

	promise, err := bs.GetBlocks(ctx, []u.Key{k})
	if err != nil {
		return nil, err
	}

	select {
	case block, ok := <-promise:
		if !ok {
			select {
			case <-ctx.Done():
				return nil, ctx.Err()
			default:
				return nil, errors.New("promise channel was closed")
			}
		}
		return block, nil
	case <-parent.Done():
		return nil, parent.Err()
	}
}

func (bs *Bitswap) WantlistForPeer(p peer.ID) []u.Key {
	var out []u.Key
	for _, e := range bs.engine.WantlistForPeer(p) {
		out = append(out, e.Key)
	}
	return out
}

// GetBlocks returns a channel where the caller may receive blocks that
// correspond to the provided |keys|. Returns an error if BitSwap is unable to
// begin this request within the deadline enforced by the context.
//
// NB: Your request remains open until the context expires. To conserve
// resources, provide a context with a reasonably short deadline (ie. not one
// that lasts throughout the lifetime of the server)
func (bs *Bitswap) GetBlocks(ctx context.Context, keys []u.Key) (<-chan *blocks.Block, error) {
	select {
	case <-bs.process.Closing():
		return nil, errors.New("bitswap is closed")
	default:
	}
	promise := bs.notifications.Subscribe(ctx, keys...)

	bs.wm.WantBlocks(keys)

	req := &blockRequest{
		keys: keys,
		ctx:  ctx,
	}
	select {
	case bs.findKeys <- req:
		return promise, nil
	case <-ctx.Done():
		return nil, ctx.Err()
	}
}

// HasBlock announces the existance of a block to this bitswap service. The
// service will potentially notify its peers.
func (bs *Bitswap) HasBlock(ctx context.Context, blk *blocks.Block) error {
	select {
	case <-bs.process.Closing():
		return errors.New("bitswap is closed")
	default:
	}

	if err := bs.blockstore.Put(blk); err != nil {
		return err
	}

	bs.notifications.Publish(blk)
	select {
	case bs.newBlocks <- blk:
		// send block off to be reprovided
	case <-ctx.Done():
		return ctx.Err()
	}
	return nil
}

func (bs *Bitswap) connectToProviders(ctx context.Context, entries []wantlist.Entry) {

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	// Get providers for all entries in wantlist (could take a while)
	wg := sync.WaitGroup{}
	for _, e := range entries {
		wg.Add(1)
		go func(k u.Key) {
			defer wg.Done()

			child, cancel := context.WithTimeout(ctx, providerRequestTimeout)
			defer cancel()
			providers := bs.network.FindProvidersAsync(child, k, maxProvidersPerRequest)
			for prov := range providers {
				go func(p peer.ID) {
					bs.network.ConnectTo(ctx, p)
				}(prov)
			}
		}(e.Key)
	}

	wg.Wait() // make sure all our children do finish.
}

func (bs *Bitswap) ReceiveMessage(ctx context.Context, p peer.ID, incoming bsmsg.BitSwapMessage) {
	// This call records changes to wantlists, blocks received,
	// and number of bytes transfered.
	bs.engine.MessageReceived(p, incoming)
	// TODO: this is bad, and could be easily abused.
	// Should only track *useful* messages in ledger

	iblocks := incoming.Blocks()

	if len(iblocks) == 0 {
		return
	}

	// quickly send out cancels, reduces chances of duplicate block receives
	var keys []u.Key
	for _, block := range iblocks {
		keys = append(keys, block.Key())
	}
	bs.wm.CancelWants(keys)

	for _, block := range iblocks {
		bs.counterLk.Lock()
		bs.blocksRecvd++
		has, err := bs.blockstore.Has(block.Key())
		if err == nil && has {
			bs.dupBlocksRecvd++
		}
		brecvd := bs.blocksRecvd
		bdup := bs.dupBlocksRecvd
		bs.counterLk.Unlock()
		if has {
			continue
		}

		// put this after the duplicate check as a block not on our wantlist may
		// have already been received.
		if _, found := bs.wm.wl.Contains(block.Key()); !found {
			log.Notice("received un-asked-for block: %s", block)
			continue
		}

		log.Infof("got block %s from %s (%d,%d)", block, p, brecvd, bdup)

		hasBlockCtx, cancel := context.WithTimeout(ctx, hasBlockTimeout)
		if err := bs.HasBlock(hasBlockCtx, block); err != nil {
			log.Warningf("ReceiveMessage HasBlock error: %s", err)
		}
		cancel()
	}
}

// Connected/Disconnected warns bitswap about peer connections
func (bs *Bitswap) PeerConnected(p peer.ID) {
	bs.wm.Connected(p)
}

// Connected/Disconnected warns bitswap about peer connections
func (bs *Bitswap) PeerDisconnected(p peer.ID) {
	bs.wm.Disconnected(p)
	bs.engine.PeerDisconnected(p)
}

func (bs *Bitswap) ReceiveError(err error) {
	log.Infof("Bitswap ReceiveError: %s", err)
	// TODO log the network error
	// TODO bubble the network error up to the parent context/error logger
}

func (bs *Bitswap) Close() error {
	return bs.process.Close()
}

func (bs *Bitswap) GetWantlist() []u.Key {
	var out []u.Key
	for _, e := range bs.wm.wl.Entries() {
		out = append(out, e.Key)
	}
	return out
}
<commit_msg>parallelize block processing<commit_after>// package bitswap implements the IPFS Exchange interface with the BitSwap
// bilateral exchange protocol.
package bitswap

import (
	"errors"
	"math"
	"sync"
	"time"

	process "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess"
	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
	blocks "github.com/ipfs/go-ipfs/blocks"
	blockstore "github.com/ipfs/go-ipfs/blocks/blockstore"
	exchange "github.com/ipfs/go-ipfs/exchange"
	decision "github.com/ipfs/go-ipfs/exchange/bitswap/decision"
	bsmsg "github.com/ipfs/go-ipfs/exchange/bitswap/message"
	bsnet "github.com/ipfs/go-ipfs/exchange/bitswap/network"
	notifications "github.com/ipfs/go-ipfs/exchange/bitswap/notifications"
	wantlist "github.com/ipfs/go-ipfs/exchange/bitswap/wantlist"
	peer "github.com/ipfs/go-ipfs/p2p/peer"
	"github.com/ipfs/go-ipfs/thirdparty/delay"
	eventlog "github.com/ipfs/go-ipfs/thirdparty/eventlog"
	u "github.com/ipfs/go-ipfs/util"
)

var log = eventlog.Logger("bitswap")

const (
	// maxProvidersPerRequest specifies the maximum number of providers desired
	// from the network. This value is specified because the network streams
	// results.
	// TODO: if a 'non-nice' strategy is implemented, consider increasing this value
	maxProvidersPerRequest = 3
	providerRequestTimeout = time.Second * 10
	hasBlockTimeout        = time.Second * 15
	provideTimeout         = time.Second * 15
	sizeBatchRequestChan   = 32
	// kMaxPriority is the max priority as defined by the bitswap protocol
	kMaxPriority = math.MaxInt32

	HasBlockBufferSize = 256
	provideWorkers     = 4
)

var rebroadcastDelay = delay.Fixed(time.Second * 10)

// New initializes a BitSwap instance that communicates over the provided
// BitSwapNetwork. This function registers the returned instance as the network
// delegate.
// Runs until context is cancelled.
func New(parent context.Context, p peer.ID, network bsnet.BitSwapNetwork,
	bstore blockstore.Blockstore, nice bool) exchange.Interface {

	// important to use provided parent context (since it may include important
	// loggable data). It's probably not a good idea to allow bitswap to be
	// coupled to the concerns of the IPFS daemon in this way.
	//
	// FIXME(btc) Now that bitswap manages itself using a process, it probably
	// shouldn't accept a context anymore. Clients should probably use Close()
	// exclusively. We should probably find another way to share logging data
	ctx, cancelFunc := context.WithCancel(parent)

	notif := notifications.New()
	px := process.WithTeardown(func() error {
		notif.Shutdown()
		return nil
	})

	go func() {
		<-px.Closing() // process closes first
		cancelFunc()
	}()
	go func() {
		<-ctx.Done() // parent cancelled first
		px.Close()
	}()

	bs := &Bitswap{
		self:          p,
		blockstore:    bstore,
		notifications: notif,
		engine:        decision.NewEngine(ctx, bstore), // TODO close the engine with Close() method
		network:       network,
		findKeys:      make(chan *blockRequest, sizeBatchRequestChan),
		process:       px,
		newBlocks:     make(chan *blocks.Block, HasBlockBufferSize),
		provideKeys:   make(chan u.Key),
		wm:            NewWantManager(ctx, network),
	}
	go bs.wm.Run()
	network.SetDelegate(bs)

	// Start up bitswaps async worker routines
	bs.startWorkers(px, ctx)
	return bs
}

// Bitswap instances implement the bitswap protocol.
type Bitswap struct {

	// the ID of the peer to act on behalf of
	self peer.ID

	// network delivers messages on behalf of the session
	network bsnet.BitSwapNetwork

	// the peermanager manages sending messages to peers in a way that
	// wont block bitswap operation
	wm *WantManager

	// blockstore is the local database
	// NB: ensure threadsafety
	blockstore blockstore.Blockstore

	notifications notifications.PubSub

	// send keys to a worker to find and connect to providers for them
	findKeys chan *blockRequest

	engine *decision.Engine

	process process.Process

	newBlocks chan *blocks.Block

	provideKeys chan u.Key

	counterLk      sync.Mutex
	blocksRecvd    int
	dupBlocksRecvd int
}

type blockRequest struct {
	keys []u.Key
	ctx  context.Context
}

// GetBlock attempts to retrieve a particular block from peers within the
// deadline enforced by the context.
func (bs *Bitswap) GetBlock(parent context.Context, k u.Key) (*blocks.Block, error) {

	// Any async work initiated by this function must end when this function
	// returns. To ensure this, derive a new context. Note that it is okay to
	// listen on parent in this scope, but NOT okay to pass |parent| to
	// functions called by this one. Otherwise those functions won't return
	// when this context's cancel func is executed. This is difficult to
	// enforce. May this comment keep you safe.

	ctx, cancelFunc := context.WithCancel(parent)

	ctx = eventlog.ContextWithLoggable(ctx, eventlog.Uuid("GetBlockRequest"))
	defer log.EventBegin(ctx, "GetBlockRequest", &k).Done()

	defer func() {
		cancelFunc()
	}()

	promise, err := bs.GetBlocks(ctx, []u.Key{k})
	if err != nil {
		return nil, err
	}

	select {
	case block, ok := <-promise:
		if !ok {
			select {
			case <-ctx.Done():
				return nil, ctx.Err()
			default:
				return nil, errors.New("promise channel was closed")
			}
		}
		return block, nil
	case <-parent.Done():
		return nil, parent.Err()
	}
}

func (bs *Bitswap) WantlistForPeer(p peer.ID) []u.Key {
	var out []u.Key
	for _, e := range bs.engine.WantlistForPeer(p) {
		out = append(out, e.Key)
	}
	return out
}

// GetBlocks returns a channel where the caller may receive blocks that
// correspond to the provided |keys|. Returns an error if BitSwap is unable to
// begin this request within the deadline enforced by the context.
//
// NB: Your request remains open until the context expires. To conserve
// resources, provide a context with a reasonably short deadline (ie. not one
// that lasts throughout the lifetime of the server)
func (bs *Bitswap) GetBlocks(ctx context.Context, keys []u.Key) (<-chan *blocks.Block, error) {
	select {
	case <-bs.process.Closing():
		return nil, errors.New("bitswap is closed")
	default:
	}
	promise := bs.notifications.Subscribe(ctx, keys...)

	bs.wm.WantBlocks(keys)

	req := &blockRequest{
		keys: keys,
		ctx:  ctx,
	}
	select {
	case bs.findKeys <- req:
		return promise, nil
	case <-ctx.Done():
		return nil, ctx.Err()
	}
}

// HasBlock announces the existance of a block to this bitswap service. The
// service will potentially notify its peers.
func (bs *Bitswap) HasBlock(ctx context.Context, blk *blocks.Block) error {
	select {
	case <-bs.process.Closing():
		return errors.New("bitswap is closed")
	default:
	}

	if err := bs.blockstore.Put(blk); err != nil {
		return err
	}

	bs.notifications.Publish(blk)
	select {
	case bs.newBlocks <- blk:
		// send block off to be reprovided
	case <-ctx.Done():
		return ctx.Err()
	}
	return nil
}

func (bs *Bitswap) connectToProviders(ctx context.Context, entries []wantlist.Entry) {

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	// Get providers for all entries in wantlist (could take a while)
	wg := sync.WaitGroup{}
	for _, e := range entries {
		wg.Add(1)
		go func(k u.Key) {
			defer wg.Done()

			child, cancel := context.WithTimeout(ctx, providerRequestTimeout)
			defer cancel()
			providers := bs.network.FindProvidersAsync(child, k, maxProvidersPerRequest)
			for prov := range providers {
				go func(p peer.ID) {
					bs.network.ConnectTo(ctx, p)
				}(prov)
			}
		}(e.Key)
	}

	wg.Wait() // make sure all our children do finish.
}

func (bs *Bitswap) ReceiveMessage(ctx context.Context, p peer.ID, incoming bsmsg.BitSwapMessage) {
	// This call records changes to wantlists, blocks received,
	// and number of bytes transfered.
	bs.engine.MessageReceived(p, incoming)
	// TODO: this is bad, and could be easily abused.
	// Should only track *useful* messages in ledger

	iblocks := incoming.Blocks()

	if len(iblocks) == 0 {
		return
	}

	// quickly send out cancels, reduces chances of duplicate block receives
	var keys []u.Key
	for _, block := range iblocks {
		if _, found := bs.wm.wl.Contains(block.Key()); !found {
			log.Notice("received un-asked-for block: %s", block)
			continue
		}
		keys = append(keys, block.Key())
	}
	bs.wm.CancelWants(keys)

	wg := sync.WaitGroup{}
	for _, block := range iblocks {
		wg.Add(1)
		go func(b *blocks.Block) {
			defer wg.Done()
			bs.counterLk.Lock()
			bs.blocksRecvd++
			has, err := bs.blockstore.Has(b.Key())
			if err == nil && has {
				bs.dupBlocksRecvd++
			}
			brecvd := bs.blocksRecvd
			bdup := bs.dupBlocksRecvd
			bs.counterLk.Unlock()
			if has {
				return
			}

			log.Debugf("got block %s from %s (%d,%d)", b, p, brecvd, bdup)
			hasBlockCtx, cancel := context.WithTimeout(ctx, hasBlockTimeout)
			if err := bs.HasBlock(hasBlockCtx, b); err != nil {
				log.Warningf("ReceiveMessage HasBlock error: %s", err)
			}
			cancel()
		}(block)
	}
	wg.Wait()
}

// Connected/Disconnected warns bitswap about peer connections
func (bs *Bitswap) PeerConnected(p peer.ID) {
	bs.wm.Connected(p)
}

// Connected/Disconnected warns bitswap about peer connections
func (bs *Bitswap) PeerDisconnected(p peer.ID) {
	bs.wm.Disconnected(p)
	bs.engine.PeerDisconnected(p)
}

func (bs *Bitswap) ReceiveError(err error) {
	log.Infof("Bitswap ReceiveError: %s", err)
	// TODO log the network error
	// TODO bubble the network error up to the parent context/error logger
}

func (bs *Bitswap) Close() error {
	return bs.process.Close()
}

func (bs *Bitswap) GetWantlist() []u.Key {
	var out []u.Key
	for _, e := range bs.wm.wl.Entries() {
		out = append(out, e.Key)
	}
	return out
}
<|endoftext|>


<commit_before>package hangoutjson

import (
	"encoding/json"
	"time"
	"strconv"
)

func Do(bytestream []byte) (Hangouts, int, error) {
	var jsontree Hangouts
	err := json.Unmarshal(bytestream, &jsontree)
	return jsontree, len(jsontree.ConversationState), err
}

func ConvTimestamp(timestamp string) (time.Time) {
	secs, err := strconv.ParseInt(timestamp[0:10], 10, 64)
	if err != nil {
		secs = 0
	}
	msecs, err := strconv.ParseInt(timestamp[10:16], 10, 64)
	if err != nil {
		msecs = 0
	}
	return time.Unix(secs,msecs)
}
<commit_msg>Add comments for routines<commit_after>package hangoutjson

import (
	"encoding/json"
	"time"
	"strconv"
)

// Do returns unmarshaled Hangouts stucture
// from bytesteam (i.e. file)
func Do(bytestream []byte) (Hangouts, int, error) {
	var jsontree Hangouts
	err := json.Unmarshal(bytestream, &jsontree)
	return jsontree, len(jsontree.ConversationState), err
}

// ConvTimestamp converts a 16 digit timestamp to a time.
func ConvTimestamp(timestamp string) (time.Time) {
	secs, err := strconv.ParseInt(timestamp[0:10], 10, 64)
	if err != nil {
		secs = 0
	}
	msecs, err := strconv.ParseInt(timestamp[10:16], 10, 64)
	if err != nil {
		msecs = 0
	}
	return time.Unix(secs,msecs)
}
<|endoftext|>