<commit_before>// SPDX-License-Identifier: Apache-2.0
// Copyright Authors of Cilium

package egressgateway

import (
	"context"
	"fmt"
	"net"
	"sort"
	"time"

	"github.com/sirupsen/logrus"
	"k8s.io/apimachinery/pkg/types"

	"github.com/cilium/cilium/pkg/identity"
	identityCache "github.com/cilium/cilium/pkg/identity/cache"
	k8sTypes "github.com/cilium/cilium/pkg/k8s/types"
	"github.com/cilium/cilium/pkg/labels"
	"github.com/cilium/cilium/pkg/lock"
	"github.com/cilium/cilium/pkg/logging"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/maps/egressmap"
	nodeTypes "github.com/cilium/cilium/pkg/node/types"
	"github.com/cilium/cilium/pkg/option"
)

var (
	log = logging.DefaultLogger.WithField(logfields.LogSubsys, "egressgateway")
)

type k8sCacheSyncedChecker interface {
	K8sCacheIsSynced() bool
}

// The egressgateway manager stores the internal data tracking the node, policy,
// endpoint, and lease mappings. It also hooks up all the callbacks to update
// egress bpf policy map accordingly.
type Manager struct {
	mutex lock.Mutex

	// k8sCacheSyncedChecker is used to check if the agent has synced its
	// cache with the k8s API server
	k8sCacheSyncedChecker k8sCacheSyncedChecker

	// nodeDataStore stores node name to node mapping
	nodeDataStore map[string]nodeTypes.Node

	// nodes stores nodes sorted by their name
	nodes []nodeTypes.Node

	// policyConfigs stores policy configs indexed by policyID
	policyConfigs map[policyID]*PolicyConfig

	// epDataStore stores endpointId to endpoint metadata mapping
	epDataStore map[endpointID]*endpointMetadata

	// identityAllocator is used to fetch identity labels for endpoint updates
	identityAllocator identityCache.IdentityAllocator
}

// NewEgressGatewayManager returns a new Egress Gateway Manager.
func NewEgressGatewayManager(k8sCacheSyncedChecker k8sCacheSyncedChecker, identityAlocator identityCache.IdentityAllocator) *Manager {
	manager := &Manager{
		k8sCacheSyncedChecker: k8sCacheSyncedChecker,
		nodeDataStore:         make(map[string]nodeTypes.Node),
		policyConfigs:         make(map[policyID]*PolicyConfig),
		epDataStore:           make(map[endpointID]*endpointMetadata),
		identityAllocator:     identityAlocator,
	}

	manager.runReconciliationAfterK8sSync()

	return manager
}

// getIdentityLabels waits for the global identities to be populated to the cache,
// then looks up identity by ID from the cached identity allocator and return its labels.
func (manager *Manager) getIdentityLabels(securityIdentity uint32) (labels.Labels, error) {
	identityCtx, cancel := context.WithTimeout(context.Background(), option.Config.KVstoreConnectivityTimeout)
	defer cancel()
	if err := manager.identityAllocator.WaitForInitialGlobalIdentities(identityCtx); err != nil {
		return nil, fmt.Errorf("failed to wait for initial global identities: %v", err)
	}

	identity := manager.identityAllocator.LookupIdentityByID(identityCtx, identity.NumericIdentity(securityIdentity))
	if identity == nil {
		return nil, fmt.Errorf("identity %d not found", securityIdentity)
	}
	return identity.Labels, nil
}

// runReconciliationAfterK8sSync spawns a goroutine that waits for the agent to
// sync with k8s and then runs the first reconciliation.
func (manager *Manager) runReconciliationAfterK8sSync() {
	go func() {
		for {
			if manager.k8sCacheSyncedChecker.K8sCacheIsSynced() {
				break
			}

			time.Sleep(1 * time.Second)
		}

		manager.mutex.Lock()
		manager.reconcile()
		manager.mutex.Unlock()
	}()
}

// Event handlers

// OnAddEgressPolicy parses the given policy config, and updates internal state
// with the config fields.
func (manager *Manager) OnAddEgressPolicy(config PolicyConfig) {
	manager.mutex.Lock()
	defer manager.mutex.Unlock()

	logger := log.WithField(logfields.CiliumEgressNATPolicyName, config.id.Name)

	if _, ok := manager.policyConfigs[config.id]; !ok {
		logger.Info("Added CiliumEgressNATPolicy")
	} else {
		logger.Info("Updated CiliumEgressNATPolicy")
	}

	manager.policyConfigs[config.id] = &config

	manager.reconcile()
}

// OnDeleteEgressPolicy deletes the internal state associated with the given
// policy, including egress eBPF map entries.
func (manager *Manager) OnDeleteEgressPolicy(configID policyID) {
	manager.mutex.Lock()
	defer manager.mutex.Unlock()

	logger := log.WithField(logfields.CiliumEgressNATPolicyName, configID.Name)

	if manager.policyConfigs[configID] == nil {
		logger.Warn("Can't delete CiliumEgressNATPolicy: policy not found")
		return
	}

	logger.Info("Deleted CiliumEgressNATPolicy")

	delete(manager.policyConfigs, configID)

	manager.reconcile()
}

// OnUpdateEndpoint is the event handler for endpoint additions and updates.
func (manager *Manager) OnUpdateEndpoint(endpoint *k8sTypes.CiliumEndpoint) {
	var epData *endpointMetadata
	var err error
	var identityLabels labels.Labels

	manager.mutex.Lock()
	defer manager.mutex.Unlock()

	logger := log.WithFields(logrus.Fields{
		logfields.K8sEndpointName: endpoint.Name,
		logfields.K8sNamespace:    endpoint.Namespace,
	})

	if len(endpoint.Networking.Addressing) == 0 {
		logger.WithError(err).
			Error("Failed to get valid endpoint IPs, skipping update to egress policy.")
		return
	}

	if identityLabels, err = manager.getIdentityLabels(uint32(endpoint.Identity.ID)); err != nil {
		logger.WithError(err).
			Error("Failed to get idenity labels for endpoint, skipping update to egress policy.")
		return
	}

	if epData, err = getEndpointMetadata(endpoint, identityLabels); err != nil {
		logger.WithError(err).
			Error("Failed to get valid endpoint metadata, skipping update to egress policy.")
		return
	}

	manager.epDataStore[epData.id] = epData

	manager.reconcile()
}

// OnDeleteEndpoint is the event handler for endpoint deletions.
func (manager *Manager) OnDeleteEndpoint(endpoint *k8sTypes.CiliumEndpoint) {
	manager.mutex.Lock()
	defer manager.mutex.Unlock()

	id := types.NamespacedName{
		Name:      endpoint.GetName(),
		Namespace: endpoint.GetNamespace(),
	}

	delete(manager.epDataStore, id)

	manager.reconcile()
}

// OnUpdateNode is the event handler for node additions and updates.
func (manager *Manager) OnUpdateNode(node nodeTypes.Node) {
	manager.mutex.Lock()
	defer manager.mutex.Unlock()
	manager.nodeDataStore[node.Name] = node
	manager.onChangeNodeLocked()
}

// OnDeleteNode is the event handler for node deletions.
func (manager *Manager) OnDeleteNode(node nodeTypes.Node) {
	manager.mutex.Lock()
	defer manager.mutex.Unlock()
	delete(manager.nodeDataStore, node.Name)
	manager.onChangeNodeLocked()
}

func (manager *Manager) onChangeNodeLocked() {
	manager.nodes = []nodeTypes.Node{}
	for _, n := range manager.nodeDataStore {
		manager.nodes = append(manager.nodes, n)
	}
	sort.Slice(manager.nodes, func(i, j int) bool {
		return manager.nodes[i].Name < manager.nodes[j].Name
	})
	manager.reconcile()
}

func (manager *Manager) regenerateGatewayConfigs() {
	for _, policyConfig := range manager.policyConfigs {
		policyConfig.regenerateGatewayConfig(manager)
	}
}

func (manager *Manager) addMissingEgressRules() {
	egressPolicies := map[egressmap.EgressPolicyKey4]egressmap.EgressPolicyVal4{}
	egressmap.EgressPolicyMap.IterateWithCallback(
		func(key *egressmap.EgressPolicyKey4, val *egressmap.EgressPolicyVal4) {
			egressPolicies[*key] = *val
		})

	addEgressRule := func(endpointIP net.IP, dstCIDR *net.IPNet, gwc *gatewayConfig) {
		policyKey := egressmap.NewEgressPolicyKey4(endpointIP, dstCIDR.IP, dstCIDR.Mask)
		policyVal, policyPresent := egressPolicies[policyKey]

		if policyPresent && policyVal.Match(gwc.egressIP.IP, gwc.gatewayIP) {
			return
		}

		logger := log.WithFields(logrus.Fields{
			logfields.SourceIP:        endpointIP,
			logfields.DestinationCIDR: dstCIDR.String(),
			logfields.EgressIP:        gwc.egressIP.IP,
			logfields.GatewayIP:       gwc.gatewayIP,
		})

		if err := egressmap.EgressPolicyMap.Update(endpointIP, *dstCIDR, gwc.egressIP.IP, gwc.gatewayIP); err != nil {
			logger.WithError(err).Error("Error applying egress gateway policy")
		} else {
			logger.Info("Egress gateway policy applied")
		}
	}

	for _, policyConfig := range manager.policyConfigs {
		policyConfig.forEachEndpointAndDestination(manager.epDataStore, addEgressRule)
	}
}

// removeUnusedEgressRules is responsible for removing any entry in the egress policy BPF map which
// is not baked by an actual k8s CiliumEgressNATPolicy.
func (manager *Manager) removeUnusedEgressRules() {
	egressPolicies := map[egressmap.EgressPolicyKey4]egressmap.EgressPolicyVal4{}
	egressmap.EgressPolicyMap.IterateWithCallback(
		func(key *egressmap.EgressPolicyKey4, val *egressmap.EgressPolicyVal4) {
			egressPolicies[*key] = *val
		})

nextPolicyKey:
	for policyKey, policyVal := range egressPolicies {
		matchPolicy := func(endpointIP net.IP, dstCIDR *net.IPNet, gwc *gatewayConfig) bool {
			return policyKey.Match(endpointIP, dstCIDR) && policyVal.Match(gwc.egressIP.IP, gwc.gatewayIP)
		}

		for _, policyConfig := range manager.policyConfigs {
			if policyConfig.matches(manager.epDataStore, matchPolicy) {
				continue nextPolicyKey
			}
		}

		logger := log.WithFields(logrus.Fields{
			logfields.SourceIP:        policyKey.GetSourceIP(),
			logfields.DestinationCIDR: policyKey.GetDestCIDR().String(),
			logfields.EgressIP:        policyVal.GetEgressIP(),
			logfields.GatewayIP:       policyVal.GetGatewayIP(),
		})

		if err := egressmap.EgressPolicyMap.Delete(policyKey.GetSourceIP(), *policyKey.GetDestCIDR()); err != nil {
			logger.WithError(err).Error("Error removing egress gateway policy")
		} else {
			logger.Info("Egress gateway policy removed")
		}
	}
}

// reconcile is responsible for reconciling the state of the manager (i.e. the
// desired state) with the actual state of the node (egress policy map entries).
//
// Whenever it encounters an error, it will just log it and move to the next
// item, in order to reconcile as many states as possible.
func (manager *Manager) reconcile() {
	if !manager.k8sCacheSyncedChecker.K8sCacheIsSynced() {
		return
	}

	manager.regenerateGatewayConfigs()

	// The order of the next 2 function calls matters, as by first adding missing policies and
	// only then removing obsolete ones we make sure there will be no connectivity disruption
	manager.addMissingEgressRules()
	manager.removeUnusedEgressRules()
}
<commit_msg>egressgw: manager: make mutex an embedded field<commit_after>// SPDX-License-Identifier: Apache-2.0
// Copyright Authors of Cilium

package egressgateway

import (
	"context"
	"fmt"
	"net"
	"sort"
	"time"

	"github.com/sirupsen/logrus"
	"k8s.io/apimachinery/pkg/types"

	"github.com/cilium/cilium/pkg/identity"
	identityCache "github.com/cilium/cilium/pkg/identity/cache"
	k8sTypes "github.com/cilium/cilium/pkg/k8s/types"
	"github.com/cilium/cilium/pkg/labels"
	"github.com/cilium/cilium/pkg/lock"
	"github.com/cilium/cilium/pkg/logging"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/maps/egressmap"
	nodeTypes "github.com/cilium/cilium/pkg/node/types"
	"github.com/cilium/cilium/pkg/option"
)

var (
	log = logging.DefaultLogger.WithField(logfields.LogSubsys, "egressgateway")
)

type k8sCacheSyncedChecker interface {
	K8sCacheIsSynced() bool
}

// The egressgateway manager stores the internal data tracking the node, policy,
// endpoint, and lease mappings. It also hooks up all the callbacks to update
// egress bpf policy map accordingly.
type Manager struct {
	lock.Mutex

	// k8sCacheSyncedChecker is used to check if the agent has synced its
	// cache with the k8s API server
	k8sCacheSyncedChecker k8sCacheSyncedChecker

	// nodeDataStore stores node name to node mapping
	nodeDataStore map[string]nodeTypes.Node

	// nodes stores nodes sorted by their name
	nodes []nodeTypes.Node

	// policyConfigs stores policy configs indexed by policyID
	policyConfigs map[policyID]*PolicyConfig

	// epDataStore stores endpointId to endpoint metadata mapping
	epDataStore map[endpointID]*endpointMetadata

	// identityAllocator is used to fetch identity labels for endpoint updates
	identityAllocator identityCache.IdentityAllocator
}

// NewEgressGatewayManager returns a new Egress Gateway Manager.
func NewEgressGatewayManager(k8sCacheSyncedChecker k8sCacheSyncedChecker, identityAlocator identityCache.IdentityAllocator) *Manager {
	manager := &Manager{
		k8sCacheSyncedChecker: k8sCacheSyncedChecker,
		nodeDataStore:         make(map[string]nodeTypes.Node),
		policyConfigs:         make(map[policyID]*PolicyConfig),
		epDataStore:           make(map[endpointID]*endpointMetadata),
		identityAllocator:     identityAlocator,
	}

	manager.runReconciliationAfterK8sSync()

	return manager
}

// getIdentityLabels waits for the global identities to be populated to the cache,
// then looks up identity by ID from the cached identity allocator and return its labels.
func (manager *Manager) getIdentityLabels(securityIdentity uint32) (labels.Labels, error) {
	identityCtx, cancel := context.WithTimeout(context.Background(), option.Config.KVstoreConnectivityTimeout)
	defer cancel()
	if err := manager.identityAllocator.WaitForInitialGlobalIdentities(identityCtx); err != nil {
		return nil, fmt.Errorf("failed to wait for initial global identities: %v", err)
	}

	identity := manager.identityAllocator.LookupIdentityByID(identityCtx, identity.NumericIdentity(securityIdentity))
	if identity == nil {
		return nil, fmt.Errorf("identity %d not found", securityIdentity)
	}
	return identity.Labels, nil
}

// runReconciliationAfterK8sSync spawns a goroutine that waits for the agent to
// sync with k8s and then runs the first reconciliation.
func (manager *Manager) runReconciliationAfterK8sSync() {
	go func() {
		for {
			if manager.k8sCacheSyncedChecker.K8sCacheIsSynced() {
				break
			}

			time.Sleep(1 * time.Second)
		}

		manager.Lock()
		manager.reconcile()
		manager.Unlock()
	}()
}

// Event handlers

// OnAddEgressPolicy parses the given policy config, and updates internal state
// with the config fields.
func (manager *Manager) OnAddEgressPolicy(config PolicyConfig) {
	manager.Lock()
	defer manager.Unlock()

	logger := log.WithField(logfields.CiliumEgressNATPolicyName, config.id.Name)

	if _, ok := manager.policyConfigs[config.id]; !ok {
		logger.Info("Added CiliumEgressNATPolicy")
	} else {
		logger.Info("Updated CiliumEgressNATPolicy")
	}

	manager.policyConfigs[config.id] = &config

	manager.reconcile()
}

// OnDeleteEgressPolicy deletes the internal state associated with the given
// policy, including egress eBPF map entries.
func (manager *Manager) OnDeleteEgressPolicy(configID policyID) {
	manager.Lock()
	defer manager.Unlock()

	logger := log.WithField(logfields.CiliumEgressNATPolicyName, configID.Name)

	if manager.policyConfigs[configID] == nil {
		logger.Warn("Can't delete CiliumEgressNATPolicy: policy not found")
		return
	}

	logger.Info("Deleted CiliumEgressNATPolicy")

	delete(manager.policyConfigs, configID)

	manager.reconcile()
}

// OnUpdateEndpoint is the event handler for endpoint additions and updates.
func (manager *Manager) OnUpdateEndpoint(endpoint *k8sTypes.CiliumEndpoint) {
	var epData *endpointMetadata
	var err error
	var identityLabels labels.Labels

	manager.Lock()
	defer manager.Unlock()

	logger := log.WithFields(logrus.Fields{
		logfields.K8sEndpointName: endpoint.Name,
		logfields.K8sNamespace:    endpoint.Namespace,
	})

	if len(endpoint.Networking.Addressing) == 0 {
		logger.WithError(err).
			Error("Failed to get valid endpoint IPs, skipping update to egress policy.")
		return
	}

	if identityLabels, err = manager.getIdentityLabels(uint32(endpoint.Identity.ID)); err != nil {
		logger.WithError(err).
			Error("Failed to get idenity labels for endpoint, skipping update to egress policy.")
		return
	}

	if epData, err = getEndpointMetadata(endpoint, identityLabels); err != nil {
		logger.WithError(err).
			Error("Failed to get valid endpoint metadata, skipping update to egress policy.")
		return
	}

	manager.epDataStore[epData.id] = epData

	manager.reconcile()
}

// OnDeleteEndpoint is the event handler for endpoint deletions.
func (manager *Manager) OnDeleteEndpoint(endpoint *k8sTypes.CiliumEndpoint) {
	manager.Lock()
	defer manager.Unlock()

	id := types.NamespacedName{
		Name:      endpoint.GetName(),
		Namespace: endpoint.GetNamespace(),
	}

	delete(manager.epDataStore, id)

	manager.reconcile()
}

// OnUpdateNode is the event handler for node additions and updates.
func (manager *Manager) OnUpdateNode(node nodeTypes.Node) {
	manager.Lock()
	defer manager.Unlock()
	manager.nodeDataStore[node.Name] = node
	manager.onChangeNodeLocked()
}

// OnDeleteNode is the event handler for node deletions.
func (manager *Manager) OnDeleteNode(node nodeTypes.Node) {
	manager.Lock()
	defer manager.Unlock()
	delete(manager.nodeDataStore, node.Name)
	manager.onChangeNodeLocked()
}

func (manager *Manager) onChangeNodeLocked() {
	manager.nodes = []nodeTypes.Node{}
	for _, n := range manager.nodeDataStore {
		manager.nodes = append(manager.nodes, n)
	}
	sort.Slice(manager.nodes, func(i, j int) bool {
		return manager.nodes[i].Name < manager.nodes[j].Name
	})
	manager.reconcile()
}

func (manager *Manager) regenerateGatewayConfigs() {
	for _, policyConfig := range manager.policyConfigs {
		policyConfig.regenerateGatewayConfig(manager)
	}
}

func (manager *Manager) addMissingEgressRules() {
	egressPolicies := map[egressmap.EgressPolicyKey4]egressmap.EgressPolicyVal4{}
	egressmap.EgressPolicyMap.IterateWithCallback(
		func(key *egressmap.EgressPolicyKey4, val *egressmap.EgressPolicyVal4) {
			egressPolicies[*key] = *val
		})

	addEgressRule := func(endpointIP net.IP, dstCIDR *net.IPNet, gwc *gatewayConfig) {
		policyKey := egressmap.NewEgressPolicyKey4(endpointIP, dstCIDR.IP, dstCIDR.Mask)
		policyVal, policyPresent := egressPolicies[policyKey]

		if policyPresent && policyVal.Match(gwc.egressIP.IP, gwc.gatewayIP) {
			return
		}

		logger := log.WithFields(logrus.Fields{
			logfields.SourceIP:        endpointIP,
			logfields.DestinationCIDR: dstCIDR.String(),
			logfields.EgressIP:        gwc.egressIP.IP,
			logfields.GatewayIP:       gwc.gatewayIP,
		})

		if err := egressmap.EgressPolicyMap.Update(endpointIP, *dstCIDR, gwc.egressIP.IP, gwc.gatewayIP); err != nil {
			logger.WithError(err).Error("Error applying egress gateway policy")
		} else {
			logger.Info("Egress gateway policy applied")
		}
	}

	for _, policyConfig := range manager.policyConfigs {
		policyConfig.forEachEndpointAndDestination(manager.epDataStore, addEgressRule)
	}
}

// removeUnusedEgressRules is responsible for removing any entry in the egress policy BPF map which
// is not baked by an actual k8s CiliumEgressNATPolicy.
func (manager *Manager) removeUnusedEgressRules() {
	egressPolicies := map[egressmap.EgressPolicyKey4]egressmap.EgressPolicyVal4{}
	egressmap.EgressPolicyMap.IterateWithCallback(
		func(key *egressmap.EgressPolicyKey4, val *egressmap.EgressPolicyVal4) {
			egressPolicies[*key] = *val
		})

nextPolicyKey:
	for policyKey, policyVal := range egressPolicies {
		matchPolicy := func(endpointIP net.IP, dstCIDR *net.IPNet, gwc *gatewayConfig) bool {
			return policyKey.Match(endpointIP, dstCIDR) && policyVal.Match(gwc.egressIP.IP, gwc.gatewayIP)
		}

		for _, policyConfig := range manager.policyConfigs {
			if policyConfig.matches(manager.epDataStore, matchPolicy) {
				continue nextPolicyKey
			}
		}

		logger := log.WithFields(logrus.Fields{
			logfields.SourceIP:        policyKey.GetSourceIP(),
			logfields.DestinationCIDR: policyKey.GetDestCIDR().String(),
			logfields.EgressIP:        policyVal.GetEgressIP(),
			logfields.GatewayIP:       policyVal.GetGatewayIP(),
		})

		if err := egressmap.EgressPolicyMap.Delete(policyKey.GetSourceIP(), *policyKey.GetDestCIDR()); err != nil {
			logger.WithError(err).Error("Error removing egress gateway policy")
		} else {
			logger.Info("Egress gateway policy removed")
		}
	}
}

// reconcile is responsible for reconciling the state of the manager (i.e. the
// desired state) with the actual state of the node (egress policy map entries).
//
// Whenever it encounters an error, it will just log it and move to the next
// item, in order to reconcile as many states as possible.
func (manager *Manager) reconcile() {
	if !manager.k8sCacheSyncedChecker.K8sCacheIsSynced() {
		return
	}

	manager.regenerateGatewayConfigs()

	// The order of the next 2 function calls matters, as by first adding missing policies and
	// only then removing obsolete ones we make sure there will be no connectivity disruption
	manager.addMissingEgressRules()
	manager.removeUnusedEgressRules()
}
<|endoftext|>


<commit_before>// Copyright 2016 DeepFabric, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package pdserver

import (
	"time"

	"github.com/deepfabric/elasticell/pkg/pb/pdpb"
	. "github.com/pingcap/check"
)

type testWatcherSuite struct {
}

func (s *testWatcherSuite) SetUpSuite(c *C) {

}

func (s *testWatcherSuite) TearDownSuite(c *C) {

}

func (s *testWatcherSuite) TestAddWatcher(c *C) {
	addr := "a"
	wn := newWatcherNotifier(time.Millisecond * 100)
	wn.addWatcher(addr)
	c.Assert(len(wn.watchers) == 1, IsTrue)
	c.Assert(wn.watchers[addr].state == ready, IsTrue)
	time.Sleep(time.Millisecond * 150)
	c.Assert(len(wn.watchers) == 1, IsTrue)
	c.Assert(wn.watchers[addr].state == paused, IsTrue)

	c.Assert(wn.watcherHeartbeat(addr, 0), IsTrue)
	c.Assert(len(wn.watchers) == 1, IsTrue)
	c.Assert(wn.watchers[addr].state == ready, IsTrue)

	time.Sleep(time.Millisecond * 80)
	wn.watcherHeartbeat(addr, 0)
	c.Assert(len(wn.watchers) == 1, IsTrue)
	c.Assert(wn.watchers[addr].state == ready, IsTrue)
}

func (s *testWatcherSuite) TestRemoveWatcher(c *C) {
	addr := "a"
	wn := newWatcherNotifier(time.Millisecond * 100)
	wn.addWatcher(addr)
	wn.removeWatcher(addr)
	c.Assert(len(wn.watchers) == 0, IsTrue)

	time.Sleep(time.Millisecond * 150)
	c.Assert(len(wn.watchers) == 0, IsTrue)

	c.Assert(wn.watcherHeartbeat(addr, 0), IsFalse)
	c.Assert(len(wn.watchers) == 0, IsTrue)
}

func (s *testWatcherSuite) TestNotify(c *C) {
	addr := "a"
	wn := newWatcherNotifier(time.Second * 100)
	wn.addWatcher(addr)
	c.Assert(wn.watchers[addr].state == ready, IsTrue)
	wn.notify(new(pdpb.Range))
	c.Assert(wn.watchers[addr].q.GetMaxOffset() == 0, IsTrue)
	wn.notify(new(pdpb.Range))
	c.Assert(wn.watchers[addr].q.GetMaxOffset() == 1, IsTrue)

	// pause
	wn.pause(addr)
	time.Sleep(time.Millisecond * 150)
	c.Assert(wn.watchers[addr].state == paused, IsTrue)

	// resume
	wn.watcherHeartbeat(addr, 0)
	c.Assert(wn.watchers[addr].state == ready, IsTrue)
	wn.notify(new(pdpb.Range))
	wn.notify(new(pdpb.Range))
	wn.notify(new(pdpb.Range))
	c.Assert(wn.watchers[addr].q.GetMaxOffset() == 2, IsTrue)
}

func newTestNotify(offset uint64, addr string) *notify {
	return &notify{
		offset:  offset,
		watcher: addr,
	}
}
<commit_msg>fix: test case<commit_after>// Copyright 2016 DeepFabric, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package pdserver

import (
	"time"

	"github.com/deepfabric/elasticell/pkg/pb/pdpb"
	. "github.com/pingcap/check"
)

type testWatcherSuite struct {
}

func (s *testWatcherSuite) SetUpSuite(c *C) {

}

func (s *testWatcherSuite) TearDownSuite(c *C) {

}

func (s *testWatcherSuite) TestAddWatcher(c *C) {
	addr := "a"
	wn := newWatcherNotifier(time.Millisecond * 100)
	wn.addWatcher(addr)
	c.Assert(len(wn.watchers) == 1, IsTrue)
	c.Assert(wn.watchers[addr].state == ready, IsTrue)
	time.Sleep(time.Millisecond * 150)
	c.Assert(len(wn.watchers) == 1, IsTrue)
	c.Assert(wn.watchers[addr].state == paused, IsTrue)

	c.Assert(wn.watcherHeartbeat(addr, 0), IsTrue)
	c.Assert(len(wn.watchers) == 1, IsTrue)
	c.Assert(wn.watchers[addr].state == ready, IsTrue)

	time.Sleep(time.Millisecond * 80)
	wn.watcherHeartbeat(addr, 0)
	c.Assert(len(wn.watchers) == 1, IsTrue)
	c.Assert(wn.watchers[addr].state == ready, IsTrue)
}

func (s *testWatcherSuite) TestRemoveWatcher(c *C) {
	addr := "a"
	wn := newWatcherNotifier(time.Millisecond * 100)
	wn.addWatcher(addr)
	wn.removeWatcher(addr)
	c.Assert(len(wn.watchers) == 0, IsTrue)

	time.Sleep(time.Millisecond * 150)
	c.Assert(len(wn.watchers) == 0, IsTrue)

	c.Assert(wn.watcherHeartbeat(addr, 0), IsFalse)
	c.Assert(len(wn.watchers) == 0, IsTrue)
}

func (s *testWatcherSuite) TestNotify(c *C) {
	addr := "a"
	wn := newWatcherNotifier(time.Second * 100)
	wn.addWatcher(addr)
	c.Assert(wn.watchers[addr].state == ready, IsTrue)
	wn.notify(new(pdpb.Range))
	c.Assert(wn.watchers[addr].q.GetMaxOffset() == 0, IsTrue)
	wn.notify(new(pdpb.Range))
	c.Assert(wn.watchers[addr].q.GetMaxOffset() == 1, IsTrue)

	// pause
	wn.pause(addr, false)
	time.Sleep(time.Millisecond * 150)
	c.Assert(wn.watchers[addr].state == paused, IsTrue)

	// resume
	wn.watcherHeartbeat(addr, 0)
	c.Assert(wn.watchers[addr].state == ready, IsTrue)
	wn.notify(new(pdpb.Range))
	wn.notify(new(pdpb.Range))
	wn.notify(new(pdpb.Range))
	c.Assert(wn.watchers[addr].q.GetMaxOffset() == 2, IsTrue)
}

func newTestNotify(offset uint64, addr string) *notify {
	return &notify{
		offset:  offset,
		watcher: addr,
	}
}
<|endoftext|>


<commit_before>package donut

import (
	"bytes"
	"io"
	"strconv"
	"time"

	"crypto/md5"
	"encoding/hex"
	"errors"
	"github.com/minio-io/minio/pkg/encoding/erasure"
	"github.com/minio-io/minio/pkg/utils/split"
	"strings"
)

// erasureReader - returns aligned streaming reads over a PipeWriter
func erasureReader(readers []io.ReadCloser, donutMetadata map[string]string, writer *io.PipeWriter) {
	// TODO handle errors
	totalChunks, _ := strconv.Atoi(donutMetadata["chunkCount"])
	totalLeft, _ := strconv.Atoi(donutMetadata["size"])
	blockSize, _ := strconv.Atoi(donutMetadata["blockSize"])
	k, _ := strconv.Atoi(donutMetadata["erasureK"])
	m, _ := strconv.Atoi(donutMetadata["erasureM"])
	expectedMd5sum, _ := hex.DecodeString(donutMetadata["md5"])
	summer := md5.New()
	// TODO select technique properly
	params, _ := erasure.ParseEncoderParams(uint8(k), uint8(m), erasure.Cauchy)
	encoder := erasure.NewEncoder(params)
	for _, reader := range readers {
		defer reader.Close()
	}
	for i := 0; i < totalChunks; i++ {
		curBlockSize := totalLeft
		if blockSize < totalLeft {
			curBlockSize = blockSize
		}
		curChunkSize := erasure.GetEncodedChunkLen(curBlockSize, uint8(k))

		encodedBytes := make([][]byte, 16)
		for i, reader := range readers {
			var bytesBuffer bytes.Buffer
			// TODO watch for errors
			io.CopyN(&bytesBuffer, reader, int64(curChunkSize))
			encodedBytes[i] = bytesBuffer.Bytes()
		}
		decodedData, err := encoder.Decode(encodedBytes, curBlockSize)
		if err != nil {
			writer.CloseWithError(err)
			return
		}
		summer.Write(decodedData)
		io.Copy(writer, bytes.NewBuffer(decodedData))
		totalLeft = totalLeft - blockSize
	}
	actualMd5sum := summer.Sum(nil)
	if bytes.Compare(expectedMd5sum, actualMd5sum) != 0 {
		writer.CloseWithError(errors.New("decoded md5sum did not match"))
		return
	}
	writer.Close()
}

// erasure writer

type erasureWriter struct {
	writers       []Writer
	metadata      map[string]string
	donutMetadata map[string]string // not exposed
	erasureWriter *io.PipeWriter
	isClosed      <-chan bool
}

// newErasureWriter - get a new writer
func newErasureWriter(writers []Writer) ObjectWriter {
	r, w := io.Pipe()
	isClosed := make(chan bool)
	writer := erasureWriter{
		writers:       writers,
		metadata:      make(map[string]string),
		erasureWriter: w,
		isClosed:      isClosed,
	}
	go erasureGoroutine(r, writer, isClosed)
	return writer
}

func erasureGoroutine(r *io.PipeReader, eWriter erasureWriter, isClosed chan<- bool) {
	chunks := split.Stream(r, 10*1024*1024)
	params, _ := erasure.ParseEncoderParams(8, 8, erasure.Cauchy)
	encoder := erasure.NewEncoder(params)
	chunkCount := 0
	totalLength := 0
	summer := md5.New()
	for chunk := range chunks {
		if chunk.Err == nil {
			totalLength = totalLength + len(chunk.Data)
			encodedBlocks, _ := encoder.Encode(chunk.Data)
			summer.Write(chunk.Data)
			for blockIndex, block := range encodedBlocks {
				io.Copy(eWriter.writers[blockIndex], bytes.NewBuffer(block))
			}
		}
		chunkCount = chunkCount + 1
	}
	dataMd5sum := summer.Sum(nil)
	metadata := make(map[string]string)
	metadata["blockSize"] = strconv.Itoa(10 * 1024 * 1024)
	metadata["chunkCount"] = strconv.Itoa(chunkCount)
	metadata["created"] = time.Now().Format(time.RFC3339Nano)
	metadata["erasureK"] = "8"
	metadata["erasureM"] = "8"
	metadata["erasureTechnique"] = "Cauchy"
	metadata["md5"] = hex.EncodeToString(dataMd5sum)
	metadata["size"] = strconv.Itoa(totalLength)
	for _, nodeWriter := range eWriter.writers {
		if nodeWriter != nil {
			nodeWriter.SetMetadata(eWriter.metadata)
			nodeWriter.SetDonutMetadata(metadata)
			nodeWriter.Close()
		}
	}
	isClosed <- true
}

func (eWriter erasureWriter) Write(data []byte) (int, error) {
	io.Copy(eWriter.erasureWriter, bytes.NewBuffer(data))
	return len(data), nil
}

func (eWriter erasureWriter) Close() error {
	eWriter.erasureWriter.Close()
	<-eWriter.isClosed
	return nil
}

func (eWriter erasureWriter) CloseWithError(err error) error {
	for _, writer := range eWriter.writers {
		if writer != nil {
			writer.CloseWithError(err)
		}
	}
	return nil
}

func (eWriter erasureWriter) SetMetadata(metadata map[string]string) error {
	for k := range metadata {
		if strings.HasPrefix(k, "sys.") {
			return errors.New("Invalid key '" + k + "', cannot start with sys.'")
		}
	}
	for k := range eWriter.metadata {
		delete(eWriter.metadata, k)
	}
	for k, v := range metadata {
		eWriter.metadata[k] = v
	}
	return nil
}

func (eWriter erasureWriter) GetMetadata() (map[string]string, error) {
	metadata := make(map[string]string)
	for k, v := range eWriter.metadata {
		metadata[k] = v
	}
	return metadata, nil
}
<commit_msg>Handle errors properly during erasure Decoding, also populate technique and verify<commit_after>package donut

import (
	"bytes"
	"errors"
	"io"
	"strconv"
	"strings"
	"time"

	"crypto/md5"
	"encoding/hex"

	"github.com/minio-io/minio/pkg/encoding/erasure"
	"github.com/minio-io/minio/pkg/utils/split"
)

// getErasureTechnique - convert technique string into Technique type
func getErasureTechnique(technique string) (erasure.Technique, error) {
	switch true {
	case technique == "Cauchy":
		return erasure.Cauchy, nil
	case technique == "Vandermonde":
		return erasure.Cauchy, nil
	default:
		return -1, errors.New("Invalid erasure technique")
	}
}

// erasureReader - returns aligned streaming reads over a PipeWriter
func erasureReader(readers []io.ReadCloser, donutMetadata map[string]string, writer *io.PipeWriter) {
	totalChunks, err := strconv.Atoi(donutMetadata["chunkCount"])
	if err != nil {
		writer.CloseWithError(err)
		return
	}
	totalLeft, err := strconv.Atoi(donutMetadata["size"])
	if err != nil {
		writer.CloseWithError(err)
		return
	}
	blockSize, err := strconv.Atoi(donutMetadata["blockSize"])
	if err != nil {
		writer.CloseWithError(err)
		return
	}
	k, err := strconv.Atoi(donutMetadata["erasureK"])
	if err != nil {
		writer.CloseWithError(err)
		return
	}
	m, err := strconv.Atoi(donutMetadata["erasureM"])
	if err != nil {
		writer.CloseWithError(err)
		return
	}
	expectedMd5sum, err := hex.DecodeString(donutMetadata["md5"])
	if err != nil {
		writer.CloseWithError(err)
		return
	}
	technique, err := getErasureTechnique(donutMetadata["erasureTechnique"])
	if err != nil {
		writer.CloseWithError(err)
		return
	}
	summer := md5.New()
	params, _ := erasure.ParseEncoderParams(uint8(k), uint8(m), technique)
	encoder := erasure.NewEncoder(params)
	for _, reader := range readers {
		defer reader.Close()
	}
	for i := 0; i < totalChunks; i++ {
		curBlockSize := totalLeft
		if blockSize < totalLeft {
			curBlockSize = blockSize
		}
		curChunkSize := erasure.GetEncodedChunkLen(curBlockSize, uint8(k))
		encodedBytes := make([][]byte, 16)
		for i, reader := range readers {
			var bytesBuffer bytes.Buffer
			_, err := io.CopyN(&bytesBuffer, reader, int64(curChunkSize))
			if err != nil {
				writer.CloseWithError(err)
				return
			}
			encodedBytes[i] = bytesBuffer.Bytes()
		}
		decodedData, err := encoder.Decode(encodedBytes, curBlockSize)
		if err != nil {
			writer.CloseWithError(err)
			return
		}
		summer.Write(decodedData)
		_, err = io.Copy(writer, bytes.NewBuffer(decodedData))
		if err != nil {
			writer.CloseWithError(err)
			return
		}
		totalLeft = totalLeft - blockSize
	}
	actualMd5sum := summer.Sum(nil)
	if bytes.Compare(expectedMd5sum, actualMd5sum) != 0 {
		writer.CloseWithError(errors.New("decoded md5sum did not match"))
		return
	}
	writer.Close()
	return
}

// erasure writer

type erasureWriter struct {
	writers       []Writer
	metadata      map[string]string
	donutMetadata map[string]string // not exposed
	erasureWriter *io.PipeWriter
	isClosed      <-chan bool
}

// newErasureWriter - get a new writer
func newErasureWriter(writers []Writer) ObjectWriter {
	r, w := io.Pipe()
	isClosed := make(chan bool)
	writer := erasureWriter{
		writers:       writers,
		metadata:      make(map[string]string),
		erasureWriter: w,
		isClosed:      isClosed,
	}
	go erasureGoroutine(r, writer, isClosed)
	return writer
}

func erasureGoroutine(r *io.PipeReader, eWriter erasureWriter, isClosed chan<- bool) {
	chunks := split.Stream(r, 10*1024*1024)
	params, _ := erasure.ParseEncoderParams(8, 8, erasure.Cauchy)
	encoder := erasure.NewEncoder(params)
	chunkCount := 0
	totalLength := 0
	summer := md5.New()
	for chunk := range chunks {
		if chunk.Err == nil {
			totalLength = totalLength + len(chunk.Data)
			encodedBlocks, _ := encoder.Encode(chunk.Data)
			summer.Write(chunk.Data)
			for blockIndex, block := range encodedBlocks {
				io.Copy(eWriter.writers[blockIndex], bytes.NewBuffer(block))
			}
		}
		chunkCount = chunkCount + 1
	}
	dataMd5sum := summer.Sum(nil)
	metadata := make(map[string]string)
	metadata["blockSize"] = strconv.Itoa(10 * 1024 * 1024)
	metadata["chunkCount"] = strconv.Itoa(chunkCount)
	metadata["created"] = time.Now().Format(time.RFC3339Nano)
	metadata["erasureK"] = "8"
	metadata["erasureM"] = "8"
	metadata["erasureTechnique"] = "Cauchy"
	metadata["md5"] = hex.EncodeToString(dataMd5sum)
	metadata["size"] = strconv.Itoa(totalLength)
	for _, nodeWriter := range eWriter.writers {
		if nodeWriter != nil {
			nodeWriter.SetMetadata(eWriter.metadata)
			nodeWriter.SetDonutMetadata(metadata)
			nodeWriter.Close()
		}
	}
	isClosed <- true
}

func (eWriter erasureWriter) Write(data []byte) (int, error) {
	io.Copy(eWriter.erasureWriter, bytes.NewBuffer(data))
	return len(data), nil
}

func (eWriter erasureWriter) Close() error {
	eWriter.erasureWriter.Close()
	<-eWriter.isClosed
	return nil
}

func (eWriter erasureWriter) CloseWithError(err error) error {
	for _, writer := range eWriter.writers {
		if writer != nil {
			writer.CloseWithError(err)
		}
	}
	return nil
}

func (eWriter erasureWriter) SetMetadata(metadata map[string]string) error {
	for k := range metadata {
		if strings.HasPrefix(k, "sys.") {
			return errors.New("Invalid key '" + k + "', cannot start with sys.'")
		}
	}
	for k := range eWriter.metadata {
		delete(eWriter.metadata, k)
	}
	for k, v := range metadata {
		eWriter.metadata[k] = v
	}
	return nil
}

func (eWriter erasureWriter) GetMetadata() (map[string]string, error) {
	metadata := make(map[string]string)
	for k, v := range eWriter.metadata {
		metadata[k] = v
	}
	return metadata, nil
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/codeskyblue/go-sh"
)

// ShellCmd represents a shell command to be run for dokku
type ShellCmd struct {
	Env           map[string]string
	Command       *exec.Cmd
	CommandString string
	Args          []string
	ShowOutput    bool
}

// NewShellCmd returns a new ShellCmd struct
func NewShellCmd(command string) *ShellCmd {
	items := strings.Split(command, " ")
	cmd := items[0]
	args := items[1:]
	return NewShellCmdWithArgs(cmd, args...)
}

// NewShellCmdWithArgs returns a new ShellCmd struct
func NewShellCmdWithArgs(cmd string, args ...string) *ShellCmd {
	commandString := strings.Join(append([]string{cmd}, args...), " ")

	return &ShellCmd{
		Command:       exec.Command(cmd, args...),
		CommandString: commandString,
		Args:          args,
		ShowOutput:    true,
	}
}

func (sc *ShellCmd) setup() {
	env := os.Environ()
	for k, v := range sc.Env {
		env = append(env, fmt.Sprintf("%s=%s", k, v))
	}
	sc.Command.Env = env
	if sc.ShowOutput {
		sc.Command.Stdout = os.Stdout
		sc.Command.Stderr = os.Stderr
	}
}

// Execute is a lightweight wrapper around exec.Command
func (sc *ShellCmd) Execute() bool {
	sc.setup()

	if err := sc.Command.Run(); err != nil {
		return false
	}
	return true
}

// Start is a wrapper around exec.Command.Start()
func (sc *ShellCmd) Start() error {
	sc.setup()

	return sc.Command.Start()
}

// Output is a lightweight wrapper around exec.Command.Output()
func (sc *ShellCmd) Output() ([]byte, error) {
	sc.setup()
	return sc.Command.Output()
}

// CombinedOutput is a lightweight wrapper around exec.Command.CombinedOutput()
func (sc *ShellCmd) CombinedOutput() ([]byte, error) {
	sc.setup()
	return sc.Command.CombinedOutput()
}

// PlugnTrigger fire the given plugn trigger with the given args
func PlugnTrigger(triggerName string, args ...string) error {
	LogDebug(fmt.Sprintf("plugn trigger %s %v", triggerName, args))
	return PlugnTriggerSetup(triggerName, args...).Run()
}

// PlugnTriggerOutput fire the given plugn trigger with the given args
func PlugnTriggerOutput(triggerName string, args ...string) ([]byte, error) {
	LogDebug(fmt.Sprintf("plugn trigger %s %v", triggerName, args))
	rE, wE, _ := os.Pipe()
	rO, wO, _ := os.Pipe()
	session := PlugnTriggerSetup(triggerName, args...)
	session.Stderr = wE
	session.Stdout = wO
	err := session.Run()
	wE.Close()
	wO.Close()

	readStderr, _ := ioutil.ReadAll(rE)
	readStdout, _ := ioutil.ReadAll(rO)

	stderr := string(readStderr[:])
	if err != nil {
		err = fmt.Errorf(stderr)
	}

	if os.Getenv("DOKKU_TRACE") == "1" {
		for _, line := range strings.Split(stderr, "\n") {
			LogDebug(fmt.Sprintf("plugn trigger %s stderr: %s", triggerName, line))
		}
		for _, line := range strings.Split(string(readStdout[:]), "\n") {
			LogDebug(fmt.Sprintf("plugn trigger %s stdout: %s", triggerName, line))
		}
	}

	return readStdout, err
}

// PlugnTriggerSetup sets up a plugn trigger call
func PlugnTriggerSetup(triggerName string, args ...string) *sh.Session {
	shellArgs := make([]interface{}, len(args)+2)
	shellArgs[0] = "trigger"
	shellArgs[1] = triggerName
	for i, arg := range args {
		shellArgs[i+2] = arg
	}
	return sh.Command("plugn", shellArgs...)
}

// PlugnTriggerExists returns whether a plugin trigger exists (ignoring the existence of any within the 20_events plugin)
func PlugnTriggerExists(triggerName string) bool {
	pluginPath := MustGetEnv("PLUGIN_ENABLED_PATH")
	glob := filepath.Join(pluginPath, "*", triggerName)
	exists := false
	files, _ := filepath.Glob(glob)
	for _, file := range files {
		plugin := strings.Trim(strings.TrimPrefix(strings.TrimSuffix(file, "/user-auth-app"), pluginPath), "/")
		if plugin != "20_events" {
			exists = true
			break
		}
	}
	return exists
}
<commit_msg>fix: allow check to work for any trigger<commit_after>package common

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/codeskyblue/go-sh"
)

// ShellCmd represents a shell command to be run for dokku
type ShellCmd struct {
	Env           map[string]string
	Command       *exec.Cmd
	CommandString string
	Args          []string
	ShowOutput    bool
}

// NewShellCmd returns a new ShellCmd struct
func NewShellCmd(command string) *ShellCmd {
	items := strings.Split(command, " ")
	cmd := items[0]
	args := items[1:]
	return NewShellCmdWithArgs(cmd, args...)
}

// NewShellCmdWithArgs returns a new ShellCmd struct
func NewShellCmdWithArgs(cmd string, args ...string) *ShellCmd {
	commandString := strings.Join(append([]string{cmd}, args...), " ")

	return &ShellCmd{
		Command:       exec.Command(cmd, args...),
		CommandString: commandString,
		Args:          args,
		ShowOutput:    true,
	}
}

func (sc *ShellCmd) setup() {
	env := os.Environ()
	for k, v := range sc.Env {
		env = append(env, fmt.Sprintf("%s=%s", k, v))
	}
	sc.Command.Env = env
	if sc.ShowOutput {
		sc.Command.Stdout = os.Stdout
		sc.Command.Stderr = os.Stderr
	}
}

// Execute is a lightweight wrapper around exec.Command
func (sc *ShellCmd) Execute() bool {
	sc.setup()

	if err := sc.Command.Run(); err != nil {
		return false
	}
	return true
}

// Start is a wrapper around exec.Command.Start()
func (sc *ShellCmd) Start() error {
	sc.setup()

	return sc.Command.Start()
}

// Output is a lightweight wrapper around exec.Command.Output()
func (sc *ShellCmd) Output() ([]byte, error) {
	sc.setup()
	return sc.Command.Output()
}

// CombinedOutput is a lightweight wrapper around exec.Command.CombinedOutput()
func (sc *ShellCmd) CombinedOutput() ([]byte, error) {
	sc.setup()
	return sc.Command.CombinedOutput()
}

// PlugnTrigger fire the given plugn trigger with the given args
func PlugnTrigger(triggerName string, args ...string) error {
	LogDebug(fmt.Sprintf("plugn trigger %s %v", triggerName, args))
	return PlugnTriggerSetup(triggerName, args...).Run()
}

// PlugnTriggerOutput fire the given plugn trigger with the given args
func PlugnTriggerOutput(triggerName string, args ...string) ([]byte, error) {
	LogDebug(fmt.Sprintf("plugn trigger %s %v", triggerName, args))
	rE, wE, _ := os.Pipe()
	rO, wO, _ := os.Pipe()
	session := PlugnTriggerSetup(triggerName, args...)
	session.Stderr = wE
	session.Stdout = wO
	err := session.Run()
	wE.Close()
	wO.Close()

	readStderr, _ := ioutil.ReadAll(rE)
	readStdout, _ := ioutil.ReadAll(rO)

	stderr := string(readStderr[:])
	if err != nil {
		err = fmt.Errorf(stderr)
	}

	if os.Getenv("DOKKU_TRACE") == "1" {
		for _, line := range strings.Split(stderr, "\n") {
			LogDebug(fmt.Sprintf("plugn trigger %s stderr: %s", triggerName, line))
		}
		for _, line := range strings.Split(string(readStdout[:]), "\n") {
			LogDebug(fmt.Sprintf("plugn trigger %s stdout: %s", triggerName, line))
		}
	}

	return readStdout, err
}

// PlugnTriggerSetup sets up a plugn trigger call
func PlugnTriggerSetup(triggerName string, args ...string) *sh.Session {
	shellArgs := make([]interface{}, len(args)+2)
	shellArgs[0] = "trigger"
	shellArgs[1] = triggerName
	for i, arg := range args {
		shellArgs[i+2] = arg
	}
	return sh.Command("plugn", shellArgs...)
}

// PlugnTriggerExists returns whether a plugin trigger exists (ignoring the existence of any within the 20_events plugin)
func PlugnTriggerExists(triggerName string) bool {
	pluginPath := MustGetEnv("PLUGIN_ENABLED_PATH")
	glob := filepath.Join(pluginPath, "*", triggerName)
	exists := false
	files, _ := filepath.Glob(glob)
	for _, file := range files {
		plugin := strings.Trim(strings.TrimPrefix(strings.TrimSuffix(file, "/"+triggerName), pluginPath), "/")
		if plugin != "20_events" {
			exists = true
			break
		}
	}
	return exists
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// An integration test that uses a real S3 account. Run as follows:
//
//     go run integration_test/*.go \
//         -key_id <key ID> \
//         -bucket <bucket> \
//         -region s3-ap-northeast-1.amazonaws.com
//
// Before doing this, create an empty bucket (or delete the contents of an
// existing bucket) using the S3 management console.

package main

import (
	"flag"
	"fmt"
	"github.com/jacobsa/aws"
	"github.com/jacobsa/aws/s3"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"os"
	"regexp"
	"testing"
)

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

var keyId = flag.String("key_id", "", "Access key ID.")
var bucketName = flag.String("bucket", "", "Bucket name.")
var region = flag.String("region", "", "Region endpoint server.")
var accessKey aws.AccessKey

type integrationTest struct {
}

////////////////////////////////////////////////////////////////////////
// Bucket
////////////////////////////////////////////////////////////////////////

type BucketTest struct {
	integrationTest
	bucket s3.Bucket
}

func init() { RegisterTestSuite(&BucketTest{}) }

func (t *BucketTest) SetUp(i *TestInfo) {
	var err error

	// Open a bucket.
	t.bucket, err = s3.OpenBucket(*bucketName, s3.Region(*region), accessKey)
	AssertEq(nil, err)
}

func (t *BucketTest) WrongAccessKeySecret() {
	// Open a bucket with the wrong key.
	wrongKey := accessKey
	wrongKey.Secret += "taco"

	bucket, err := s3.OpenBucket(*bucketName, s3.Region(*region), wrongKey)
	AssertEq(nil, err)

	// Attempt to do something.
	_, err = bucket.ListKeys("")
	ExpectThat(err, Error(HasSubstr("signature")))
}

func (t *BucketTest) ListEmptyBucket() {
	var keys []string
	var err error

	// From start.
	keys, err = t.bucket.ListKeys("")
	AssertEq(nil, err)
	ExpectThat(keys, ElementsAre())

	// From middle.
	keys, err = t.bucket.ListKeys("foo")
	AssertEq(nil, err)
	ExpectThat(keys, ElementsAre())
}

func (t *BucketTest) ListWithEmptyMinimum() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) ListWithInvalidUtf8Minimum() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) ListWithLongMinimum() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) ListWithNullByteInMinimum() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) ListFewKeys() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) ListManyKeys() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) GetNonExistentObject() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) StoreThenGetObject() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) StoreThenDeleteObject() {
	ExpectFalse(true, "TODO")
}

////////////////////////////////////////////////////////////////////////
// main
////////////////////////////////////////////////////////////////////////

func main() {
	flag.Parse()

	if *keyId == "" {
		fmt.Println("You must set the -key_id flag.")
		fmt.Println("Find a key ID here:")
		fmt.Println("    https://portal.aws.amazon.com/gp/aws/securityCredentials")
		os.Exit(1)
	}

	if *bucketName == "" {
		fmt.Println("You must set the -bucket flag.")
		fmt.Println("Manage your buckets here:")
		fmt.Println("    https://console.aws.amazon.com/s3/")
		os.Exit(1)
	}

	if *region == "" {
		fmt.Println("You must set the -region flag. See region.go.")
		os.Exit(1)
	}

	// Read in the access key.
	accessKey.Id = *keyId
	accessKey.Secret = readPassword("Access key secret: ")

	// Run the tests.
	matchString := func(pat, str string) (bool, error) {
		re, err := regexp.Compile(pat)
		if err != nil {
			return false, err
		}

		return re.MatchString(str), nil
	}

	testing.Main(
		matchString,
		[]testing.InternalTest{
			testing.InternalTest{
				Name: "IntegrationTest",
				F:    func(t *testing.T) { RunTests(t) },
			},
		},
		[]testing.InternalBenchmark{},
		[]testing.InternalExample{},
	)
}
<commit_msg>Re-organized.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// An integration test that uses a real S3 account. Run as follows:
//
//     go run integration_test/*.go \
//         -key_id <key ID> \
//         -bucket <bucket> \
//         -region s3-ap-northeast-1.amazonaws.com
//
// Before doing this, create an empty bucket (or delete the contents of an
// existing bucket) using the S3 management console.

package main

import (
	"flag"
	"fmt"
	"github.com/jacobsa/aws"
	"github.com/jacobsa/aws/s3"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"os"
	"regexp"
	"testing"
)

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

var keyId = flag.String("key_id", "", "Access key ID.")
var bucketName = flag.String("bucket", "", "Bucket name.")
var region = flag.String("region", "", "Region endpoint server.")
var accessKey aws.AccessKey

type integrationTest struct {
}

////////////////////////////////////////////////////////////////////////
// Bucket
////////////////////////////////////////////////////////////////////////

type BucketTest struct {
	integrationTest
	bucket s3.Bucket
}

func init() { RegisterTestSuite(&BucketTest{}) }

func (t *BucketTest) SetUp(i *TestInfo) {
	var err error

	// Open a bucket.
	t.bucket, err = s3.OpenBucket(*bucketName, s3.Region(*region), accessKey)
	AssertEq(nil, err)
}

func (t *BucketTest) WrongAccessKeySecret() {
	// Open a bucket with the wrong key.
	wrongKey := accessKey
	wrongKey.Secret += "taco"

	bucket, err := s3.OpenBucket(*bucketName, s3.Region(*region), wrongKey)
	AssertEq(nil, err)

	// Attempt to do something.
	_, err = bucket.ListKeys("")
	ExpectThat(err, Error(HasSubstr("signature")))
}

func (t *BucketTest) GetNonExistentObject() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) StoreThenGetObject() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) StoreThenDeleteObject() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) ListEmptyBucket() {
	var keys []string
	var err error

	// From start.
	keys, err = t.bucket.ListKeys("")
	AssertEq(nil, err)
	ExpectThat(keys, ElementsAre())

	// From middle.
	keys, err = t.bucket.ListKeys("foo")
	AssertEq(nil, err)
	ExpectThat(keys, ElementsAre())
}

func (t *BucketTest) ListWithEmptyMinimum() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) ListWithInvalidUtf8Minimum() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) ListWithLongMinimum() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) ListWithNullByteInMinimum() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) ListFewKeys() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) ListManyKeys() {
	ExpectFalse(true, "TODO")
}

////////////////////////////////////////////////////////////////////////
// main
////////////////////////////////////////////////////////////////////////

func main() {
	flag.Parse()

	if *keyId == "" {
		fmt.Println("You must set the -key_id flag.")
		fmt.Println("Find a key ID here:")
		fmt.Println("    https://portal.aws.amazon.com/gp/aws/securityCredentials")
		os.Exit(1)
	}

	if *bucketName == "" {
		fmt.Println("You must set the -bucket flag.")
		fmt.Println("Manage your buckets here:")
		fmt.Println("    https://console.aws.amazon.com/s3/")
		os.Exit(1)
	}

	if *region == "" {
		fmt.Println("You must set the -region flag. See region.go.")
		os.Exit(1)
	}

	// Read in the access key.
	accessKey.Id = *keyId
	accessKey.Secret = readPassword("Access key secret: ")

	// Run the tests.
	matchString := func(pat, str string) (bool, error) {
		re, err := regexp.Compile(pat)
		if err != nil {
			return false, err
		}

		return re.MatchString(str), nil
	}

	testing.Main(
		matchString,
		[]testing.InternalTest{
			testing.InternalTest{
				Name: "IntegrationTest",
				F:    func(t *testing.T) { RunTests(t) },
			},
		},
		[]testing.InternalBenchmark{},
		[]testing.InternalExample{},
	)
}
<|endoftext|>


<commit_before>package safeQueue

import (
	"testing"
)

func TestQueueCreation(t *testing.T) {
	q := New()
	if q.Len() != 0 {
		t.Errorf("Failed, invalid queue length.")
	}
	q.Destroy()
	return
}

func TestQueueLength(t *testing.T) {
	q := New()
	q.Enqueue(14)
	q.Enqueue(42)
	q.Enqueue("testing")
	q.Enqueue([]byte("Viper"))
	len := q.Len()
	if len != 4 {
		t.Errorf("Failed, invalid stack length, got %d expected 4", len)
	}
	q.Destroy()
	return
}

func TestQueueOrder(t *testing.T) {
	q := New()
	q.Enqueue(16)
	q.Enqueue(32)
	q.Enqueue(64)
	nv, ok := q.Dequeue().(int);
	if !ok {
		t.Errorf("Failed, Dequeue got wrong type")
		q.Destroy()
		return
	}
	if nv != 16 {
		t.Errorf("Failed, got incorrect value order")
		q.Destroy()
		return
	}
	q.Destroy()
	return
}

func TestSizeAfterDequeue(t *testing.T) {
	q := New()
	q.Enqueue(16)
	q.Enqueue("test")
	q.Enqueue("私は笑い男だ")
	_ = q.Dequeue()
	_ = q.Dequeue()
	_ = q.Dequeue()
	if q.Len() != 0 {
		t.Errorf("Failed, poped through entire stack, yet size is non-zero")
	}
	q.Destroy()
	return
}

func TestEmptyDequeue(t *testing.T) {
	d := New()
	v := d.Dequeue()
	if v != nil {
		t.Errorf("Empty Pop got non-nil value")
	}
	d.Destroy()
}

func TestEmptyDequeueWithValues(t *testing.T) {
	q := New()
	q.Enqueue("Thingy")
	_ = q.Dequeue()
	v := q.Dequeue()
	if v != nil {
		t.Errorf("Empty stack with values got non-nil value")
	}
	q.Destroy()
}

func BenchmarkEqualRWWithInt(b *testing.B) {
	q := New()
	write := false
	for i := 0; i < b.N; i++ {
		if q.Dequeue() == nil || write {
			q.Enqueue(i)
		} else {
			q.Dequeue()
			write = true
		}
	}
	q.Destroy()
}

func BenchmarkROnlyWithInt(b *testing.B) {
	q := New()
	nbr := b.N
	for i := 0; i < nbr; i++ {
		q.Enqueue(i)
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		_ = q.Dequeue()
	}
	q.Destroy()
}

func BenchmarkWOnlyWithInt(b *testing.B) {
	q := New()
	for i := 0; i < b.N; i++ {
		q.Enqueue(i)
	}
	q.Destroy()
}
<commit_msg>Formatted Unit Tests for safeQueue<commit_after>package safeQueue

import (
	"testing"
)

func TestQueueCreation(t *testing.T) {
	q := New()
	if q.Len() != 0 {
		t.Errorf("Failed, invalid queue length.")
	}
	q.Destroy()
	return
}

func TestQueueLength(t *testing.T) {
	q := New()
	q.Enqueue(14)
	q.Enqueue(42)
	q.Enqueue("testing")
	q.Enqueue([]byte("Viper"))
	len := q.Len()
	if len != 4 {
		t.Errorf("Failed, invalid stack length, got %d expected 4", len)
	}
	q.Destroy()
	return
}

func TestQueueOrder(t *testing.T) {
	q := New()
	q.Enqueue(16)
	q.Enqueue(32)
	q.Enqueue(64)
	nv, ok := q.Dequeue().(int)
	if !ok {
		t.Errorf("Failed, Dequeue got wrong type")
		q.Destroy()
		return
	}
	if nv != 16 {
		t.Errorf("Failed, got incorrect value order")
		q.Destroy()
		return
	}
	q.Destroy()
	return
}

func TestSizeAfterDequeue(t *testing.T) {
	q := New()
	q.Enqueue(16)
	q.Enqueue("test")
	q.Enqueue("私は笑い男だ")
	_ = q.Dequeue()
	_ = q.Dequeue()
	_ = q.Dequeue()
	if q.Len() != 0 {
		t.Errorf("Failed, poped through entire stack, yet size is non-zero")
	}
	q.Destroy()
	return
}

func TestEmptyDequeue(t *testing.T) {
	d := New()
	v := d.Dequeue()
	if v != nil {
		t.Errorf("Empty Pop got non-nil value")
	}
	d.Destroy()
}

func TestEmptyDequeueWithValues(t *testing.T) {
	q := New()
	q.Enqueue("Thingy")
	_ = q.Dequeue()
	v := q.Dequeue()
	if v != nil {
		t.Errorf("Empty stack with values got non-nil value")
	}
	q.Destroy()
}

func BenchmarkEqualRWWithInt(b *testing.B) {
	q := New()
	write := false
	for i := 0; i < b.N; i++ {
		if q.Dequeue() == nil || write {
			q.Enqueue(i)
		} else {
			q.Dequeue()
			write = true
		}
	}
	q.Destroy()
}

func BenchmarkROnlyWithInt(b *testing.B) {
	q := New()
	nbr := b.N
	for i := 0; i < nbr; i++ {
		q.Enqueue(i)
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		_ = q.Dequeue()
	}
	q.Destroy()
}

func BenchmarkWOnlyWithInt(b *testing.B) {
	q := New()
	for i := 0; i < b.N; i++ {
		q.Enqueue(i)
	}
	q.Destroy()
}
<|endoftext|>


<commit_before>package pushmessager

import (
	"appengine"
	"appengine/urlfetch"

	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"time"
	"strings"
)

type TopStoresRes struct {
	Collection []int64
}

type ItemDetails struct {
	By    string  `by`
	Id    int64   `id`
	Kids  []int64 `kids`
	Score int64   `score`
	Text  string  `text`
	Time  int64   `time`
	Title string  `title`
	Type  string  `type`
	Url   string  `url`
}

type SyncItemDetails struct {
	By          string
	Id          int64
	Kids        int
	Score       int64
	Text        string
	Time        int64
	Title       string
	Type        string
	Url         string
	Pushed_Time string
}

type SyncItemDetailsList struct {
	SyncList []SyncItemDetails
}

//Get list of all "ids" of items.
func getTopStories(w http.ResponseWriter, r *http.Request, api string) []int64 {
	apiUrl := API_HOST + API_VERSION + "/" + api + ".json"
	cxt := appengine.NewContext(r)
	if req, err := http.NewRequest(API_METHOD, apiUrl, nil); err == nil {
		httpClient := urlfetch.Client(cxt)
		r, err := httpClient.Do(req)
		defer r.Body.Close()
		if err == nil {
			if bytes, err := ioutil.ReadAll(r.Body); err == nil {
				topStoresRes := make([]int64, 0)
				json.Unmarshal(bytes, &topStoresRes)
				return topStoresRes
			} else {
				cxt.Errorf("getTopStories unmarshal: %v", err)
			}
		} else {
			cxt.Errorf("getTopStories doing: %v", err)
		}
	} else {
		cxt.Errorf("getTopStories: %v", err)
	}
	return nil
}

//Load detail of item.
func loadItemDetail(w http.ResponseWriter, r *http.Request, itemId int64, detailsList *[]*ItemDetails, ch chan int) {
	api := fmt.Sprintf(API_GET_ITEM_DETAILS, itemId)
	cxt := appengine.NewContext(r)
	if req, err := http.NewRequest(API_METHOD, api, nil); err == nil {
		httpClient := urlfetch.Client(cxt)
		res, err := httpClient.Do(req)
		if res != nil {
			defer res.Body.Close()
		}
		if err == nil {
			if bytes, err := ioutil.ReadAll(res.Body); err == nil {
				details := new(ItemDetails)
				json.Unmarshal(bytes, &details)
				*detailsList = append(*detailsList, details)
				ch <- 0
			} else {
				cxt.Errorf("loadItemDetail unmarshal: %v", err)
				ch <- 0
			}
		} else {
			cxt.Errorf("loadItemDetail doing: %v", err)
			ch <- 0
		}
	} else {
		cxt.Errorf("loadItemDetail: %v", err)
		ch <- 0
	}
}

//Get object detail.
func getItemDetails(w http.ResponseWriter, r *http.Request, itemIds []int64) []*ItemDetails {
	detailsList := []*ItemDetails{}
	ch := make(chan int)

	for _, itemId := range itemIds {
		go loadItemDetail(w, r, itemId, &detailsList, ch)
	}

	c := len(itemIds)
	for i := 0; i < c; i++ {
		<-ch
	}

	return detailsList
}

//Dispatch messages to client, do details of push.
func dispatchOnClients(w http.ResponseWriter, r *http.Request, topicApi string, pItemDetailsList *[]*ItemDetails, pushedTime string) {
	//Dispatch details to client.
	if pItemDetailsList != nil {
		c := len(*pItemDetailsList)
		ch := make(chan int, c)
		for _, pItemDetail := range *pItemDetailsList {
			if pItemDetail != nil {
				go func(w http.ResponseWriter, r *http.Request, topicApi string, pItemDetail *ItemDetails, pushedTime string, ch chan int) {
					broadcast(w, r, topicApi, pItemDetail, pushedTime)
					ch <- 0
				}(w, r, topicApi, pItemDetail, pushedTime, ch)
			}
		}
		for i := 0; i < c; i++ {
			<-ch
		}
	}
	//Push summary to client.
	endCh := make(chan int)
	go summary(w, r, pItemDetailsList, pushedTime, endCh)
	<-endCh
}

//Push messages to clients.
func push(w http.ResponseWriter, r *http.Request, api string) {
	topicApi := (TOPICS + api)
	itemDetailsList := getItemDetails(w, r, getTopStories(w, r, api))
	t := time.Now()
	pushedTime := t.Format("20060102150405")

	dispatchCh := make(chan int)

	go func(w http.ResponseWriter,
		r *http.Request,
		topicApi string,
		pItemDetailsList *[]*ItemDetails,
		pushedTime string,
		dispatchCh chan int) {

		dispatchOnClients(w, r, topicApi, pItemDetailsList, pushedTime)
		dispatchCh <- 0
	}(w, r, topicApi, &itemDetailsList, pushedTime, dispatchCh)

	<-dispatchCh
}

//Send summary to clients.
func summary(w http.ResponseWriter, r *http.Request, pItemDetailsList *[]*ItemDetails, pushedTime string, endCh chan int) {
	//Push server can not accept to long contents.
	loopCount := len(*pItemDetailsList)
	if loopCount > SUMMARY_MAX {
		loopCount = SUMMARY_MAX
	}
	msgIds := ""
	summary := ""
	for i := 0; i < loopCount; i++ {
		summary += (((*pItemDetailsList)[i]).Title + "<tr>")
		msgIds += (strconv.FormatInt(((*pItemDetailsList)[i]).Id, 10) + ",")
	}
	pushedMsg := fmt.Sprintf(`{"to" : "%s", "data" : {"isSummary" : true, "summary": "%s", "ids": "%s", "count": %d, "pushed_time" : "%s"}}`,
		TOPICS + GET_SUMMARY,
		summary,
		msgIds,
		loopCount,
		pushedTime)
	pushedMsgBytes := bytes.NewBufferString(pushedMsg)

	cxt := appengine.NewContext(r)
	if req, err := http.NewRequest("POST", PUSH_SENDER, pushedMsgBytes); err == nil {
		req.Header.Add("Authorization", PUSH_KEY)
		req.Header.Add("Content-Type", API_RESTYPE)
		req.Header.Add("X-AppEngine-Cron", "true")

		client := urlfetch.Client(cxt)
		res, _ := client.Do(req)
		if res != nil {
			defer res.Body.Close()
		}
		if err != nil {
			cxt.Errorf("Push summary doing: %v", err)
		}
	} else {
		cxt.Errorf("Push summary: %v", err)
	}
	if endCh != nil {
		endCh <- 0
	}
}

func broadcast(w http.ResponseWriter, r *http.Request, topics string, pDetails *ItemDetails, pushedTime string) {
	pushedMsg := fmt.Sprintf(
		`{"to" : "%s","data" : {"by": "%s", "c_id": %d, "score": %d, "comments_count": %d, "text": "%s", "time": %d, "title": "%s", "url": "%s", "pushed_time" : "%s"}}`,
		topics,
		pDetails.By,
		pDetails.Id,
		pDetails.Score,
		len(pDetails.Kids),
		"", //details.Text,
		pDetails.Time,
		pDetails.Title,
		pDetails.Url,
		pushedTime)
	pushedMsgBytes := bytes.NewBufferString(pushedMsg)

	cxt := appengine.NewContext(r)
	if req, err := http.NewRequest("POST", PUSH_SENDER, pushedMsgBytes); err == nil {
		req.Header.Add("Authorization", PUSH_KEY)
		req.Header.Add("Content-Type", API_RESTYPE)
		req.Header.Add("X-AppEngine-Cron", "true")

		client := urlfetch.Client(cxt)
		res, err := client.Do(req)
		if res != nil {
			defer res.Body.Close()
		}
		if err != nil {
			cxt.Errorf("Push broadcast doing: %v", err)
		}
	} else {
		cxt.Errorf("Push broadcast: %v", err)
	}
}

//Sync top-news to client.
func sync(w http.ResponseWriter, r *http.Request) {
	cxt := appengine.NewContext(r)
	itemDetailsList := getItemDetails(w, r, getTopStories(w, r, GET_TOP_STORIES))
	syncItems := []SyncItemDetails{}
	t := time.Now()
	pushedTime := t.Format("20060102150405")
	for _, itemDetail := range itemDetailsList {
		//Same logical like dispatch, but I don't use routine.
		itemDetail.Title = strings.Replace(itemDetail.Title, "\"", "'", -1)
		itemDetail.Title = strings.Replace(itemDetail.Title, "%", "％", -1)
		itemDetail.Title = strings.Replace(itemDetail.Title, "\\", ",", -1)
		syncItem := SyncItemDetails{
			By:          itemDetail.By,
			Id:          itemDetail.Id,
			Kids:        len(itemDetail.Kids),
			Score:       itemDetail.Score,
			Text:        "", //itemDetail.Text,
			Time:        itemDetail.Time,
			Title:       itemDetail.Title,
			Url:         itemDetail.Url,
			Pushed_Time: pushedTime}
		syncItems = append(syncItems, syncItem)
	}
	if len(syncItems) > 0 {
		if json, err := json.Marshal(SyncItemDetailsList{syncItems}); err == nil {
			result := string(json)
			result = strings.Replace(result, "(MISSING)","", -1)
			w.Header().Set("Content-Type", API_RESTYPE)
			fmt.Fprintf(w, result)
		} else {
			cxt.Errorf("sync marshal: %v", err)
		}
	}
}
<commit_msg>Fix bug of url.<commit_after>package pushmessager

import (
	"appengine"
	"appengine/urlfetch"

	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"time"
	"strings"
	"net/url"
)

type TopStoresRes struct {
	Collection []int64
}

type ItemDetails struct {
	By    string  `by`
	Id    int64   `id`
	Kids  []int64 `kids`
	Score int64   `score`
	Text  string  `text`
	Time  int64   `time`
	Title string  `title`
	Type  string  `type`
	Url   string  `url`
}

type SyncItemDetails struct {
	By          string
	Id          int64
	Kids        int
	Score       int64
	Text        string
	Time        int64
	Title       string
	Type        string
	Url         string
	Pushed_Time string
}

type SyncItemDetailsList struct {
	SyncList []SyncItemDetails
}

//Get list of all "ids" of items.
func getTopStories(w http.ResponseWriter, r *http.Request, api string) []int64 {
	apiUrl := API_HOST + API_VERSION + "/" + api + ".json"
	cxt := appengine.NewContext(r)
	if req, err := http.NewRequest(API_METHOD, apiUrl, nil); err == nil {
		httpClient := urlfetch.Client(cxt)
		r, err := httpClient.Do(req)
		defer r.Body.Close()
		if err == nil {
			if bytes, err := ioutil.ReadAll(r.Body); err == nil {
				topStoresRes := make([]int64, 0)
				json.Unmarshal(bytes, &topStoresRes)
				return topStoresRes
			} else {
				cxt.Errorf("getTopStories unmarshal: %v", err)
			}
		} else {
			cxt.Errorf("getTopStories doing: %v", err)
		}
	} else {
		cxt.Errorf("getTopStories: %v", err)
	}
	return nil
}

//Load detail of item.
func loadItemDetail(w http.ResponseWriter, r *http.Request, itemId int64, detailsList *[]*ItemDetails, ch chan int) {
	api := fmt.Sprintf(API_GET_ITEM_DETAILS, itemId)
	cxt := appengine.NewContext(r)
	if req, err := http.NewRequest(API_METHOD, api, nil); err == nil {
		httpClient := urlfetch.Client(cxt)
		res, err := httpClient.Do(req)
		if res != nil {
			defer res.Body.Close()
		}
		if err == nil {
			if bytes, err := ioutil.ReadAll(res.Body); err == nil {
				details := new(ItemDetails)
				json.Unmarshal(bytes, &details)
				*detailsList = append(*detailsList, details)
				ch <- 0
			} else {
				cxt.Errorf("loadItemDetail unmarshal: %v", err)
				ch <- 0
			}
		} else {
			cxt.Errorf("loadItemDetail doing: %v", err)
			ch <- 0
		}
	} else {
		cxt.Errorf("loadItemDetail: %v", err)
		ch <- 0
	}
}

//Get object detail.
func getItemDetails(w http.ResponseWriter, r *http.Request, itemIds []int64) []*ItemDetails {
	detailsList := []*ItemDetails{}
	ch := make(chan int)

	for _, itemId := range itemIds {
		go loadItemDetail(w, r, itemId, &detailsList, ch)
	}

	c := len(itemIds)
	for i := 0; i < c; i++ {
		<-ch
	}

	return detailsList
}

//Dispatch messages to client, do details of push.
func dispatchOnClients(w http.ResponseWriter, r *http.Request, topicApi string, pItemDetailsList *[]*ItemDetails, pushedTime string) {
	//Dispatch details to client.
	if pItemDetailsList != nil {
		c := len(*pItemDetailsList)
		ch := make(chan int, c)
		for _, pItemDetail := range *pItemDetailsList {
			if pItemDetail != nil {
				go func(w http.ResponseWriter, r *http.Request, topicApi string, pItemDetail *ItemDetails, pushedTime string, ch chan int) {
					broadcast(w, r, topicApi, pItemDetail, pushedTime)
					ch <- 0
				}(w, r, topicApi, pItemDetail, pushedTime, ch)
			}
		}
		for i := 0; i < c; i++ {
			<-ch
		}
	}
	//Push summary to client.
	endCh := make(chan int)
	go summary(w, r, pItemDetailsList, pushedTime, endCh)
	<-endCh
}

//Push messages to clients.
func push(w http.ResponseWriter, r *http.Request, api string) {
	topicApi := (TOPICS + api)
	itemDetailsList := getItemDetails(w, r, getTopStories(w, r, api))
	t := time.Now()
	pushedTime := t.Format("20060102150405")

	dispatchCh := make(chan int)

	go func(w http.ResponseWriter,
		r *http.Request,
		topicApi string,
		pItemDetailsList *[]*ItemDetails,
		pushedTime string,
		dispatchCh chan int) {

		dispatchOnClients(w, r, topicApi, pItemDetailsList, pushedTime)
		dispatchCh <- 0
	}(w, r, topicApi, &itemDetailsList, pushedTime, dispatchCh)

	<-dispatchCh
}

//Send summary to clients.
func summary(w http.ResponseWriter, r *http.Request, pItemDetailsList *[]*ItemDetails, pushedTime string, endCh chan int) {
	//Push server can not accept to long contents.
	loopCount := len(*pItemDetailsList)
	if loopCount > SUMMARY_MAX {
		loopCount = SUMMARY_MAX
	}
	msgIds := ""
	summary := ""
	for i := 0; i < loopCount; i++ {
		summary += (((*pItemDetailsList)[i]).Title + "<tr>")
		msgIds += (strconv.FormatInt(((*pItemDetailsList)[i]).Id, 10) + ",")
	}
	pushedMsg := fmt.Sprintf(`{"to" : "%s", "data" : {"isSummary" : true, "summary": "%s", "ids": "%s", "count": %d, "pushed_time" : "%s"}}`,
		TOPICS + GET_SUMMARY,
		summary,
		msgIds,
		loopCount,
		pushedTime)
	pushedMsgBytes := bytes.NewBufferString(pushedMsg)

	cxt := appengine.NewContext(r)
	if req, err := http.NewRequest("POST", PUSH_SENDER, pushedMsgBytes); err == nil {
		req.Header.Add("Authorization", PUSH_KEY)
		req.Header.Add("Content-Type", API_RESTYPE)
		req.Header.Add("X-AppEngine-Cron", "true")

		client := urlfetch.Client(cxt)
		res, _ := client.Do(req)
		if res != nil {
			defer res.Body.Close()
		}
		if err != nil {
			cxt.Errorf("Push summary doing: %v", err)
		}
	} else {
		cxt.Errorf("Push summary: %v", err)
	}
	if endCh != nil {
		endCh <- 0
	}
}

func broadcast(w http.ResponseWriter, r *http.Request, topics string, pDetails *ItemDetails, pushedTime string) {
	pushedMsg := fmt.Sprintf(
		`{"to" : "%s","data" : {"by": "%s", "c_id": %d, "score": %d, "comments_count": %d, "text": "%s", "time": %d, "title": "%s", "url": "%s", "pushed_time" : "%s"}}`,
		topics,
		pDetails.By,
		pDetails.Id,
		pDetails.Score,
		len(pDetails.Kids),
		"", //details.Text,
		pDetails.Time,
		pDetails.Title,
		pDetails.Url,
		pushedTime)
	pushedMsgBytes := bytes.NewBufferString(pushedMsg)

	cxt := appengine.NewContext(r)
	if req, err := http.NewRequest("POST", PUSH_SENDER, pushedMsgBytes); err == nil {
		req.Header.Add("Authorization", PUSH_KEY)
		req.Header.Add("Content-Type", API_RESTYPE)
		req.Header.Add("X-AppEngine-Cron", "true")

		client := urlfetch.Client(cxt)
		res, err := client.Do(req)
		if res != nil {
			defer res.Body.Close()
		}
		if err != nil {
			cxt.Errorf("Push broadcast doing: %v", err)
		}
	} else {
		cxt.Errorf("Push broadcast: %v", err)
	}
}

//Sync top-news to client.
func sync(w http.ResponseWriter, r *http.Request) {
	cxt := appengine.NewContext(r)
	itemDetailsList := getItemDetails(w, r, getTopStories(w, r, GET_TOP_STORIES))
	syncItems := []SyncItemDetails{}
	t := time.Now()
	pushedTime := t.Format("20060102150405")
	for _, itemDetail := range itemDetailsList {
		//Same logical like dispatch, but I don't use routine.
		itemDetail.Title = strings.Replace(itemDetail.Title, "\"", "'", -1)
		itemDetail.Title = strings.Replace(itemDetail.Title, "%", "％", -1)
		itemDetail.Title = strings.Replace(itemDetail.Title, "\\", ",", -1)
		itemDetail.Url, _ = url.QueryUnescape(	itemDetail.Url )
		syncItem := SyncItemDetails{
			By:          itemDetail.By,
			Id:          itemDetail.Id,
			Kids:        len(itemDetail.Kids),
			Score:       itemDetail.Score,
			Text:        "", //itemDetail.Text,
			Time:        itemDetail.Time,
			Title:       itemDetail.Title,
			Url:         itemDetail.Url,
			Pushed_Time: pushedTime}
		syncItems = append(syncItems, syncItem)
	}
	if len(syncItems) > 0 {
		if json, err := json.Marshal(SyncItemDetailsList{syncItems}); err == nil {
			result := fmt.Sprintf(`%s`,string(json))
			w.Header().Set("Content-Type", API_RESTYPE)
			fmt.Fprintf(w, result)
		} else {
			cxt.Errorf("sync marshal: %v", err)
		}
	}
}
<|endoftext|>


<commit_before>package resource

type (
	Field struct {
		Id    int    `json:"id"`
		Name  string `json:"name"`
		Value string `json:"value"`
	}

	CustomField struct {
		Id    int    `json:"id"`
		Name  string `json:"name"`
		Value string `json:"value"`
	}

	Resources interface {
		Sort(field string)
	}
)
<commit_msg>Removed unuesed field<commit_after>package resource

type (
	Field struct {
		Id    int    `json:"id"`
		Name  string `json:"name"`
	}

	CustomField struct {
		Id    int    `json:"id"`
		Name  string `json:"name"`
		Value string `json:"value"`
	}

	Resources interface {
		Sort(field string)
	}
)
<|endoftext|>


<commit_before>package pushmessager

import (
	"appengine"
	"appengine/urlfetch"

	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"
	"strconv"
)

type TopStoresRes struct {
	Collection []int64
}

type ItemDetails struct {
	By    string  `by`
	Id    int64   `id`
	Kids  []int64 `kids`
	Score int64   `score`
	Text  string  `text`
	Time  int64   `time`
	Title string  `title`
	Type  string  `type`
	Url   string  `url`
}

type SyncItemDetails struct {
	By          string
	Id          int64
	Kids        int
	Score       int64
	Text        string
	Time        int64
	Title       string
	Type        string
	Url         string
	Pushed_Time string
}

type SyncItemDetailsList struct {
	SyncList []SyncItemDetails
}

func getTopStories(_w http.ResponseWriter, _r *http.Request) []int64 {
	cxt := appengine.NewContext(_r)
	if req, err := http.NewRequest(API_METHOD, API_GET_TOP_STORIES, nil); err == nil {
		httpClient := urlfetch.Client(cxt)
		_r, err := httpClient.Do(req)
		defer _r.Body.Close()
		if err == nil {
			if bytes, err := ioutil.ReadAll(_r.Body); err == nil {
				topStoresRes := make([]int64, 0)
				json.Unmarshal(bytes, &topStoresRes)
				return topStoresRes
			} else {
				cxt.Errorf("getTopStories unmarshal: %v", err)
			}
		} else {
			cxt.Errorf("getTopStories doing: %v", err)
		}
	} else {
		cxt.Errorf("getTopStories: %v", err)
	}
	return nil
}

func loadDetailsList(_w http.ResponseWriter, _r *http.Request, itemId int64, detailsList *[]*ItemDetails, ch chan int) {
	api := fmt.Sprintf(API_GET_ITEM_DETAILS, itemId)
	cxt := appengine.NewContext(_r)
	if req, err := http.NewRequest(API_METHOD, api, nil); err == nil {
		httpClient := urlfetch.Client(cxt)
		res, err := httpClient.Do(req)
		if res != nil {
			defer res.Body.Close()
		}
		if err == nil {
			if bytes, err := ioutil.ReadAll(res.Body); err == nil {
				details := new(ItemDetails)
				json.Unmarshal(bytes, &details)
				*detailsList = append(*detailsList, details)
				ch <- 0
			} else {
				cxt.Errorf("loadDetailsList unmarshal: %v", err)
				ch <- 0
			}
		} else {
			cxt.Errorf("loadDetailsList doing: %v", err)
			ch <- 0
		}
	} else {
		cxt.Errorf("loadDetailsList: %v", err)
		ch <- 0
	}
}

func getItemDetails(_w http.ResponseWriter, _r *http.Request, itemIds []int64) []*ItemDetails {
	detailsList := []*ItemDetails{}
	ch := make(chan int)

	for _, itemId := range itemIds {
		go loadDetailsList(_w, _r, itemId, &detailsList, ch)
	}

	c := len(itemIds)
	for i := 0; i < c; i++ {
		<-ch
	}

	return detailsList
}

func dispatch(_w http.ResponseWriter, _r *http.Request, roundTotal *int,
	client OtherClient, itemDetail *ItemDetails, pushedTime string, scheduledTask bool, ch chan int) (brk bool) {
	brk = false
	if ch != nil {
		if *roundTotal < client.MsgCount {
			if client.FullText && len(strings.TrimSpace(itemDetail.Text)) == 0 {
				ch <- 0
			} else if !client.AllowEmptyUrl && len(strings.TrimSpace(itemDetail.Url)) == 0 {
				ch <- 0
			} else {
				(*roundTotal)++
				broadcast(_w, _r, client.PushID, itemDetail, pushedTime, scheduledTask)
				ch <- 0
			}
		} else {
			ch <- 0
		}
	} else {
		if *roundTotal < client.MsgCount {
			if client.FullText && len(strings.TrimSpace(itemDetail.Text)) == 0 {
			} else if !client.AllowEmptyUrl && len(strings.TrimSpace(itemDetail.Url)) == 0 {
			} else {
				(*roundTotal)++
				broadcast(_w, _r, client.PushID, itemDetail, pushedTime, scheduledTask)
			}
		} else {
			brk = true
		}
	}
	return
}

func dispatchOnClients(_w http.ResponseWriter, _r *http.Request, _itemDetailsList []*ItemDetails, client OtherClient, pushedTime string, scheduledTask bool,   syncType bool) {
	var roundTotal int = 0
	if !syncType {
		ch := make(chan int)
		for _, itemDetail := range _itemDetailsList {
			go dispatch(_w, _r, &roundTotal, client, itemDetail, pushedTime, scheduledTask, ch)
		}
		c := len(_itemDetailsList)
		for i := 0; i < c; i++ {
			<-ch
		}
	} else {
		for _, itemDetail := range _itemDetailsList {
			brk := dispatch(_w, _r, &roundTotal, client, itemDetail, pushedTime, scheduledTask, nil)
			if brk {
				break
			}
		}
	}

	if !syncType {
		endCh := make(chan int)
		go summary(_w, _r, client, _itemDetailsList, pushedTime, scheduledTask, endCh)
		<-endCh
	} else {
		summary(_w, _r, client, _itemDetailsList, pushedTime, scheduledTask, nil)
	}
}

func push(_w http.ResponseWriter, _r *http.Request, _clients []OtherClient, scheduledTask bool) {
	if _clients != nil {
		_itemDetailsList := getItemDetails(_w, _r, getTopStories(_w, _r))
		t := time.Now()
		pushedTime := t.Format("20060102150405")

		//Check how many clients needs PUSH.
		totalDispatch := len(_clients)
		dispatchCh := make(chan int, totalDispatch)

		//Dispatch PUSHs to clients.
		for _, client := range _clients {
			go func(w http.ResponseWriter, r *http.Request, itemDetailsList []*ItemDetails, client OtherClient, pushedTime string, scheduledTask bool,  syncType bool, dispatchCh chan int) {
				dispatchOnClients(w, r, itemDetailsList, client, pushedTime, scheduledTask, syncType)
				dispatchCh <- 0
			}(_w, _r, _itemDetailsList, client, pushedTime, scheduledTask, true, dispatchCh)
		}

		//Wait for all pushed clients.
		for i := 0; i < totalDispatch; i++ {
			<-dispatchCh
		}
	}
}

func summary(_w http.ResponseWriter, _r *http.Request, _client OtherClient, pushedDetailList []*ItemDetails, pushedTime string, scheduledTask bool, endCh chan int) {
	//Push server can not accept to long contents.
	loopCount := len(pushedDetailList)
	if loopCount > SUMMARY_MAX {
		loopCount = SUMMARY_MAX
	}
	msgIds := ""
	summary := ""
	for i := 0; i < loopCount; i++ {
		summary += (pushedDetailList[i].Title + "<tr>")
		msgIds += (strconv.FormatInt(pushedDetailList[i].Id, 10) + ",")
	}
	pushedMsg := fmt.Sprintf(`{"registration_ids" : ["%s"],"data" : {"isSummary" : true, "summary": "%s", "ids": "%s", "count": %d, "pushed_time" : "%s"}}`,
		_client.PushID,
		summary,
		msgIds,
		_client.MsgCount,
		pushedTime)
	pushedMsgBytes := bytes.NewBufferString(pushedMsg)

	cxt := appengine.NewContext(_r)
	if req, err := http.NewRequest("POST", PUSH_SENDER, pushedMsgBytes); err == nil {
		req.Header.Add("Authorization", PUSH_KEY)
		req.Header.Add("Content-Type", API_RESTYPE)
		if scheduledTask {
			req.Header.Add("X-AppEngine-Cron", "true")
		}
		client := urlfetch.Client(cxt)
		res, _ := client.Do(req)
		if res != nil {
			defer res.Body.Close()
		}
		if err != nil {
			cxt.Errorf("Push summary doing: %v", err)
		}
	} else {
		cxt.Errorf("Push summary: %v", err)
	}
	if endCh != nil {
		endCh <- 0
	}
}

func broadcast(_w http.ResponseWriter, _r *http.Request, clientIds string, details *ItemDetails, pushedTime string, scheduledTask bool) {
	pushedMsg := fmt.Sprintf(
		`{"registration_ids" : ["%s"],"data" : {"by": "%s", "c_id": %d, "score": %d, "comments_count": %d, "text": "%s", "time": %d, "title": "%s", "url": "%s", "pushed_time" : "%s"}}`,
		clientIds,
		details.By,
		details.Id,
		details.Score,
		len(details.Kids),
		"", //details.Text,
		details.Time,
		details.Title,
		details.Url,
		pushedTime)
	pushedMsgBytes := bytes.NewBufferString(pushedMsg)

	cxt := appengine.NewContext(_r)
	if req, err := http.NewRequest("POST", PUSH_SENDER, pushedMsgBytes); err == nil {
		req.Header.Add("Authorization", PUSH_KEY)
		req.Header.Add("Content-Type", API_RESTYPE)
		if scheduledTask {
			req.Header.Add("X-AppEngine-Cron", "true")
		}
		client := urlfetch.Client(cxt)
		res, err := client.Do(req)
		if res != nil {
			defer res.Body.Close()
		}
		if err != nil {
			cxt.Errorf("Push broadcast doing: %v", err)
		}
	} else {
		cxt.Errorf("Push broadcast: %v", err)
	}
}

func sync(_w http.ResponseWriter, _r *http.Request, client *OtherClient) {
	cxt := appengine.NewContext(_r)
	_itemDetailsList := getItemDetails(_w, _r, getTopStories(_w, _r))
	syncItems := []SyncItemDetails{}
	var roundTotal int = 0
	t := time.Now()
	pushedTime := t.Format("20060102150405")
	for _, itemDetail := range _itemDetailsList {
		//Same logical like dispatch, but I don't use routine.
		if roundTotal < client.MsgCount {
			if client.FullText && len(strings.TrimSpace(itemDetail.Text)) == 0 {
			} else if !client.AllowEmptyUrl && len(strings.TrimSpace(itemDetail.Url)) == 0 {
			} else {
				roundTotal++
				syncItem := SyncItemDetails{
					By:          itemDetail.By,
					Id:          itemDetail.Id,
					Kids:        len(itemDetail.Kids),
					Score:       itemDetail.Score,
					Text:        "", //itemDetail.Text,
					Time:        itemDetail.Time,
					Title:       itemDetail.Title,
					Url:         itemDetail.Url,
					Pushed_Time: pushedTime}
				syncItems = append(syncItems, syncItem)
			}
		} else {
			break
		}
	}
	if len(syncItems) > 0 {
		if json, err := json.Marshal(SyncItemDetailsList{syncItems}); err == nil {
			_w.Header().Set("Content-Type", API_RESTYPE)
			fmt.Fprintf(_w, string(json))
		} else {
			cxt.Errorf("sync marshal: %v", err)
		}
	}
}

//Deprecated, in case that we need more quota safe.
func getTinyUrl(_w http.ResponseWriter, _r *http.Request, orignalUrl string) (tingUrl string) {
	if orignalUrl != "" {
		cxt := appengine.NewContext(_r)
		if req, err := http.NewRequest(API_METHOD, TINY+orignalUrl, nil); err == nil {
			httpClient := urlfetch.Client(cxt)
			res, err := httpClient.Do(req)
			if res != nil {
				defer res.Body.Close()
			}
			if err == nil {
				if bytes, err := ioutil.ReadAll(res.Body); err == nil {
					tingUrl = string(bytes)
				} else {
					cxt.Errorf("getTinyUrl read: %v", err)
					tingUrl = orignalUrl
				}
			} else {
				cxt.Errorf("getTinyUrl doing: %v", err)
				tingUrl = orignalUrl
			}
		} else {
			cxt.Errorf("getTinyUrl: %v", err)
		}
	}
	return
}
<commit_msg>Add nil check before details being pushed.<commit_after>package pushmessager

import (
	"appengine"
	"appengine/urlfetch"

	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"
	"strconv"
)

type TopStoresRes struct {
	Collection []int64
}

type ItemDetails struct {
	By    string  `by`
	Id    int64   `id`
	Kids  []int64 `kids`
	Score int64   `score`
	Text  string  `text`
	Time  int64   `time`
	Title string  `title`
	Type  string  `type`
	Url   string  `url`
}

type SyncItemDetails struct {
	By          string
	Id          int64
	Kids        int
	Score       int64
	Text        string
	Time        int64
	Title       string
	Type        string
	Url         string
	Pushed_Time string
}

type SyncItemDetailsList struct {
	SyncList []SyncItemDetails
}

func getTopStories(_w http.ResponseWriter, _r *http.Request) []int64 {
	cxt := appengine.NewContext(_r)
	if req, err := http.NewRequest(API_METHOD, API_GET_TOP_STORIES, nil); err == nil {
		httpClient := urlfetch.Client(cxt)
		_r, err := httpClient.Do(req)
		defer _r.Body.Close()
		if err == nil {
			if bytes, err := ioutil.ReadAll(_r.Body); err == nil {
				topStoresRes := make([]int64, 0)
				json.Unmarshal(bytes, &topStoresRes)
				return topStoresRes
			} else {
				cxt.Errorf("getTopStories unmarshal: %v", err)
			}
		} else {
			cxt.Errorf("getTopStories doing: %v", err)
		}
	} else {
		cxt.Errorf("getTopStories: %v", err)
	}
	return nil
}

func loadDetailsList(_w http.ResponseWriter, _r *http.Request, itemId int64, detailsList *[]*ItemDetails, ch chan int) {
	api := fmt.Sprintf(API_GET_ITEM_DETAILS, itemId)
	cxt := appengine.NewContext(_r)
	if req, err := http.NewRequest(API_METHOD, api, nil); err == nil {
		httpClient := urlfetch.Client(cxt)
		res, err := httpClient.Do(req)
		if res != nil {
			defer res.Body.Close()
		}
		if err == nil {
			if bytes, err := ioutil.ReadAll(res.Body); err == nil {
				details := new(ItemDetails)
				json.Unmarshal(bytes, &details)
				*detailsList = append(*detailsList, details)
				ch <- 0
			} else {
				cxt.Errorf("loadDetailsList unmarshal: %v", err)
				ch <- 0
			}
		} else {
			cxt.Errorf("loadDetailsList doing: %v", err)
			ch <- 0
		}
	} else {
		cxt.Errorf("loadDetailsList: %v", err)
		ch <- 0
	}
}

func getItemDetails(_w http.ResponseWriter, _r *http.Request, itemIds []int64) []*ItemDetails {
	detailsList := []*ItemDetails{}
	ch := make(chan int)

	for _, itemId := range itemIds {
		go loadDetailsList(_w, _r, itemId, &detailsList, ch)
	}

	c := len(itemIds)
	for i := 0; i < c; i++ {
		<-ch
	}

	return detailsList
}

func dispatch(_w http.ResponseWriter, _r *http.Request, roundTotal *int,
	client OtherClient, itemDetail *ItemDetails, pushedTime string, scheduledTask bool, ch chan int) (brk bool) {
	brk = false
	if ch != nil {
		if *roundTotal < client.MsgCount {
			if client.FullText && len(strings.TrimSpace(itemDetail.Text)) == 0 {
				ch <- 0
			} else if !client.AllowEmptyUrl && len(strings.TrimSpace(itemDetail.Url)) == 0 {
				ch <- 0
			} else {
				if itemDetail != nil {
					(*roundTotal)++
					broadcast(_w, _r, client.PushID, itemDetail, pushedTime, scheduledTask)
					ch <- 0
				} else {
					ch <- 0
				}
			}
		} else {
			ch <- 0
		}
	} else {
		if *roundTotal < client.MsgCount {
			if client.FullText && len(strings.TrimSpace(itemDetail.Text)) == 0 {
			} else if !client.AllowEmptyUrl && len(strings.TrimSpace(itemDetail.Url)) == 0 {
			} else {
				if itemDetail != nil {
					(*roundTotal)++
					broadcast(_w, _r, client.PushID, itemDetail, pushedTime, scheduledTask)
				}
			}
		} else {
			brk = true
		}
	}
	return
}

func dispatchOnClients(_w http.ResponseWriter, _r *http.Request, _itemDetailsList []*ItemDetails, client OtherClient, pushedTime string, scheduledTask bool,   syncType bool) {
	var roundTotal int = 0
	if !syncType {
		ch := make(chan int)
		for _, itemDetail := range _itemDetailsList {
			go dispatch(_w, _r, &roundTotal, client, itemDetail, pushedTime, scheduledTask, ch)
		}
		c := len(_itemDetailsList)
		for i := 0; i < c; i++ {
			<-ch
		}
	} else {
		for _, itemDetail := range _itemDetailsList {
			brk := dispatch(_w, _r, &roundTotal, client, itemDetail, pushedTime, scheduledTask, nil)
			if brk {
				break
			}
		}
	}

	if !syncType {
		endCh := make(chan int)
		go summary(_w, _r, client, _itemDetailsList, pushedTime, scheduledTask, endCh)
		<-endCh
	} else {
		summary(_w, _r, client, _itemDetailsList, pushedTime, scheduledTask, nil)
	}
}

func push(_w http.ResponseWriter, _r *http.Request, _clients []OtherClient, scheduledTask bool) {
	if _clients != nil {
		_itemDetailsList := getItemDetails(_w, _r, getTopStories(_w, _r))
		t := time.Now()
		pushedTime := t.Format("20060102150405")

		//Check how many clients needs PUSH.
		totalDispatch := len(_clients)
		dispatchCh := make(chan int, totalDispatch)

		//Dispatch PUSHs to clients.
		for _, client := range _clients {
			go func(w http.ResponseWriter, r *http.Request, itemDetailsList []*ItemDetails, client OtherClient, pushedTime string, scheduledTask bool,  syncType bool, dispatchCh chan int) {
				dispatchOnClients(w, r, itemDetailsList, client, pushedTime, scheduledTask, syncType)
				dispatchCh <- 0
			}(_w, _r, _itemDetailsList, client, pushedTime, scheduledTask, true, dispatchCh)
		}

		//Wait for all pushed clients.
		for i := 0; i < totalDispatch; i++ {
			<-dispatchCh
		}
	}
}

func summary(_w http.ResponseWriter, _r *http.Request, _client OtherClient, pushedDetailList []*ItemDetails, pushedTime string, scheduledTask bool, endCh chan int) {
	//Push server can not accept to long contents.
	loopCount := len(pushedDetailList)
	if loopCount > SUMMARY_MAX {
		loopCount = SUMMARY_MAX
	}
	msgIds := ""
	summary := ""
	for i := 0; i < loopCount; i++ {
		summary += (pushedDetailList[i].Title + "<tr>")
		msgIds += (strconv.FormatInt(pushedDetailList[i].Id, 10) + ",")
	}
	pushedMsg := fmt.Sprintf(`{"registration_ids" : ["%s"],"data" : {"isSummary" : true, "summary": "%s", "ids": "%s", "count": %d, "pushed_time" : "%s"}}`,
		_client.PushID,
		summary,
		msgIds,
		_client.MsgCount,
		pushedTime)
	pushedMsgBytes := bytes.NewBufferString(pushedMsg)

	cxt := appengine.NewContext(_r)
	if req, err := http.NewRequest("POST", PUSH_SENDER, pushedMsgBytes); err == nil {
		req.Header.Add("Authorization", PUSH_KEY)
		req.Header.Add("Content-Type", API_RESTYPE)
		if scheduledTask {
			req.Header.Add("X-AppEngine-Cron", "true")
		}
		client := urlfetch.Client(cxt)
		res, _ := client.Do(req)
		if res != nil {
			defer res.Body.Close()
		}
		if err != nil {
			cxt.Errorf("Push summary doing: %v", err)
		}
	} else {
		cxt.Errorf("Push summary: %v", err)
	}
	if endCh != nil {
		endCh <- 0
	}
}

func broadcast(_w http.ResponseWriter, _r *http.Request, clientIds string, details *ItemDetails, pushedTime string, scheduledTask bool) {
	pushedMsg := fmt.Sprintf(
		`{"registration_ids" : ["%s"],"data" : {"by": "%s", "c_id": %d, "score": %d, "comments_count": %d, "text": "%s", "time": %d, "title": "%s", "url": "%s", "pushed_time" : "%s"}}`,
		clientIds,
		details.By,
		details.Id,
		details.Score,
		len(details.Kids),
		"", //details.Text,
		details.Time,
		details.Title,
		details.Url,
		pushedTime)
	pushedMsgBytes := bytes.NewBufferString(pushedMsg)

	cxt := appengine.NewContext(_r)
	if req, err := http.NewRequest("POST", PUSH_SENDER, pushedMsgBytes); err == nil {
		req.Header.Add("Authorization", PUSH_KEY)
		req.Header.Add("Content-Type", API_RESTYPE)
		if scheduledTask {
			req.Header.Add("X-AppEngine-Cron", "true")
		}
		client := urlfetch.Client(cxt)
		res, err := client.Do(req)
		if res != nil {
			defer res.Body.Close()
		}
		if err != nil {
			cxt.Errorf("Push broadcast doing: %v", err)
		}
	} else {
		cxt.Errorf("Push broadcast: %v", err)
	}
}

func sync(_w http.ResponseWriter, _r *http.Request, client *OtherClient) {
	cxt := appengine.NewContext(_r)
	_itemDetailsList := getItemDetails(_w, _r, getTopStories(_w, _r))
	syncItems := []SyncItemDetails{}
	var roundTotal int = 0
	t := time.Now()
	pushedTime := t.Format("20060102150405")
	for _, itemDetail := range _itemDetailsList {
		//Same logical like dispatch, but I don't use routine.
		if roundTotal < client.MsgCount {
			if client.FullText && len(strings.TrimSpace(itemDetail.Text)) == 0 {
			} else if !client.AllowEmptyUrl && len(strings.TrimSpace(itemDetail.Url)) == 0 {
			} else {
				roundTotal++
				syncItem := SyncItemDetails{
					By:          itemDetail.By,
					Id:          itemDetail.Id,
					Kids:        len(itemDetail.Kids),
					Score:       itemDetail.Score,
					Text:        "", //itemDetail.Text,
					Time:        itemDetail.Time,
					Title:       itemDetail.Title,
					Url:         itemDetail.Url,
					Pushed_Time: pushedTime}
				syncItems = append(syncItems, syncItem)
			}
		} else {
			break
		}
	}
	if len(syncItems) > 0 {
		if json, err := json.Marshal(SyncItemDetailsList{syncItems}); err == nil {
			_w.Header().Set("Content-Type", API_RESTYPE)
			fmt.Fprintf(_w, string(json))
		} else {
			cxt.Errorf("sync marshal: %v", err)
		}
	}
}

//Deprecated, in case that we need more quota safe.
func getTinyUrl(_w http.ResponseWriter, _r *http.Request, orignalUrl string) (tingUrl string) {
	if orignalUrl != "" {
		cxt := appengine.NewContext(_r)
		if req, err := http.NewRequest(API_METHOD, TINY+orignalUrl, nil); err == nil {
			httpClient := urlfetch.Client(cxt)
			res, err := httpClient.Do(req)
			if res != nil {
				defer res.Body.Close()
			}
			if err == nil {
				if bytes, err := ioutil.ReadAll(res.Body); err == nil {
					tingUrl = string(bytes)
				} else {
					cxt.Errorf("getTinyUrl read: %v", err)
					tingUrl = orignalUrl
				}
			} else {
				cxt.Errorf("getTinyUrl doing: %v", err)
				tingUrl = orignalUrl
			}
		} else {
			cxt.Errorf("getTinyUrl: %v", err)
		}
	}
	return
}
<|endoftext|>