<commit_before>package contentenc

import (
	"github.com/rfjakob/gocryptfs/internal/tlog"
)

// Contentenc methods that translate offsets between ciphertext and plaintext

// get the block number at plain-text offset
func (be *ContentEnc) PlainOffToBlockNo(plainOffset uint64) uint64 {
	return plainOffset / be.plainBS
}

// get the block number at ciphter-text offset
func (be *ContentEnc) CipherOffToBlockNo(cipherOffset uint64) uint64 {
	return (cipherOffset - HEADER_LEN) / be.cipherBS
}

// get ciphertext offset of block "blockNo"
func (be *ContentEnc) BlockNoToCipherOff(blockNo uint64) uint64 {
	return HEADER_LEN + blockNo*be.cipherBS
}

// get plaintext offset of block "blockNo"
func (be *ContentEnc) BlockNoToPlainOff(blockNo uint64) uint64 {
	return blockNo * be.plainBS
}

// PlainSize - calculate plaintext size from ciphertext size
func (be *ContentEnc) CipherSizeToPlainSize(cipherSize uint64) uint64 {

	// Zero sized files stay zero-sized
	if cipherSize == 0 {
		return 0
	}

	if cipherSize == HEADER_LEN {
		tlog.Warn.Printf("cipherSize %d == header size: interrupted write?\n", cipherSize)
		return 0
	}

	if cipherSize < HEADER_LEN {
		tlog.Warn.Printf("cipherSize %d < header size %d: corrupt file\n", cipherSize, HEADER_LEN)
		return 0
	}

	// Block number at last byte
	blockNo := be.CipherOffToBlockNo(cipherSize - 1)
	blockCount := blockNo + 1

	overhead := be.BlockOverhead()*blockCount + HEADER_LEN

	return cipherSize - overhead
}

// CipherSize - calculate ciphertext size from plaintext size
func (be *ContentEnc) PlainSizeToCipherSize(plainSize uint64) uint64 {

	// Block number at last byte
	blockNo := be.PlainOffToBlockNo(plainSize - 1)
	blockCount := blockNo + 1

	overhead := be.BlockOverhead()*blockCount + HEADER_LEN

	return plainSize + overhead
}

// Split a plaintext byte range into (possibly partial) blocks
// Returns an empty slice if length == 0.
func (be *ContentEnc) ExplodePlainRange(offset uint64, length uint64) []intraBlock {
	var blocks []intraBlock
	var nextBlock intraBlock
	nextBlock.fs = be

	for length > 0 {
		nextBlock.BlockNo = be.PlainOffToBlockNo(offset)
		nextBlock.Skip = offset - be.BlockNoToPlainOff(nextBlock.BlockNo)

		// Minimum of remaining plaintext data and remaining space in the block
		nextBlock.Length = MinUint64(length, be.plainBS-nextBlock.Skip)

		blocks = append(blocks, nextBlock)
		offset += nextBlock.Length
		length -= nextBlock.Length
	}
	return blocks
}

func (be *ContentEnc) BlockOverhead() uint64 {
	return be.cipherBS - be.plainBS
}

func MinUint64(x uint64, y uint64) uint64 {
	if x < y {
		return x
	}
	return y
}
<commit_msg>conentenc: handle zero-sized files in PlainSizeToCipherSize<commit_after>package contentenc

import (
	"github.com/rfjakob/gocryptfs/internal/tlog"
)

// Contentenc methods that translate offsets between ciphertext and plaintext

// get the block number at plain-text offset
func (be *ContentEnc) PlainOffToBlockNo(plainOffset uint64) uint64 {
	return plainOffset / be.plainBS
}

// get the block number at ciphter-text offset
func (be *ContentEnc) CipherOffToBlockNo(cipherOffset uint64) uint64 {
	return (cipherOffset - HEADER_LEN) / be.cipherBS
}

// get ciphertext offset of block "blockNo"
func (be *ContentEnc) BlockNoToCipherOff(blockNo uint64) uint64 {
	return HEADER_LEN + blockNo*be.cipherBS
}

// get plaintext offset of block "blockNo"
func (be *ContentEnc) BlockNoToPlainOff(blockNo uint64) uint64 {
	return blockNo * be.plainBS
}

// PlainSize - calculate plaintext size from ciphertext size
func (be *ContentEnc) CipherSizeToPlainSize(cipherSize uint64) uint64 {
	// Zero-sized files stay zero-sized
	if cipherSize == 0 {
		return 0
	}

	if cipherSize == HEADER_LEN {
		tlog.Warn.Printf("cipherSize %d == header size: interrupted write?\n", cipherSize)
		return 0
	}

	if cipherSize < HEADER_LEN {
		tlog.Warn.Printf("cipherSize %d < header size %d: corrupt file\n", cipherSize, HEADER_LEN)
		return 0
	}

	// Block number at last byte
	blockNo := be.CipherOffToBlockNo(cipherSize - 1)
	blockCount := blockNo + 1

	overhead := be.BlockOverhead()*blockCount + HEADER_LEN

	return cipherSize - overhead
}

// CipherSize - calculate ciphertext size from plaintext size
func (be *ContentEnc) PlainSizeToCipherSize(plainSize uint64) uint64 {
	// Zero-sized files stay zero-sized
	if plainSize == 0 {
		return 0
	}

	// Block number at last byte
	blockNo := be.PlainOffToBlockNo(plainSize - 1)
	blockCount := blockNo + 1

	overhead := be.BlockOverhead()*blockCount + HEADER_LEN

	return plainSize + overhead
}

// Split a plaintext byte range into (possibly partial) blocks
// Returns an empty slice if length == 0.
func (be *ContentEnc) ExplodePlainRange(offset uint64, length uint64) []intraBlock {
	var blocks []intraBlock
	var nextBlock intraBlock
	nextBlock.fs = be

	for length > 0 {
		nextBlock.BlockNo = be.PlainOffToBlockNo(offset)
		nextBlock.Skip = offset - be.BlockNoToPlainOff(nextBlock.BlockNo)

		// Minimum of remaining plaintext data and remaining space in the block
		nextBlock.Length = MinUint64(length, be.plainBS-nextBlock.Skip)

		blocks = append(blocks, nextBlock)
		offset += nextBlock.Length
		length -= nextBlock.Length
	}
	return blocks
}

func (be *ContentEnc) BlockOverhead() uint64 {
	return be.cipherBS - be.plainBS
}

func MinUint64(x uint64, y uint64) uint64 {
	if x < y {
		return x
	}
	return y
}
<|endoftext|>


<commit_before>package kubernetes

import (
	"errors"
	"fmt"
	"io/ioutil"
	"time"

	"github.com/containous/traefik/log"
	corev1 "k8s.io/api/core/v1"
	extensionsv1beta1 "k8s.io/api/extensions/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"
)

const resyncPeriod = 10 * time.Minute

type resourceEventHandler struct {
	ev chan<- interface{}
}

func (reh *resourceEventHandler) OnAdd(obj interface{}) {
	eventHandlerFunc(reh.ev, obj)
}

func (reh *resourceEventHandler) OnUpdate(oldObj, newObj interface{}) {
	eventHandlerFunc(reh.ev, newObj)
}

func (reh *resourceEventHandler) OnDelete(obj interface{}) {
	eventHandlerFunc(reh.ev, obj)
}

// Client is a client for the Provider master.
// WatchAll starts the watch of the Provider resources and updates the stores.
// The stores can then be accessed via the Get* functions.
type Client interface {
	WatchAll(namespaces Namespaces, stopCh <-chan struct{}) (<-chan interface{}, error)
	GetIngresses() []*extensionsv1beta1.Ingress
	GetService(namespace, name string) (*corev1.Service, bool, error)
	GetSecret(namespace, name string) (*corev1.Secret, bool, error)
	GetEndpoints(namespace, name string) (*corev1.Endpoints, bool, error)
	UpdateIngressStatus(namespace, name, ip, hostname string) error
}

type clientImpl struct {
	clientset            *kubernetes.Clientset
	factories            map[string]informers.SharedInformerFactory
	ingressLabelSelector labels.Selector
	isNamespaceAll       bool
}

func newClientImpl(clientset *kubernetes.Clientset) *clientImpl {
	return &clientImpl{
		clientset: clientset,
		factories: make(map[string]informers.SharedInformerFactory),
	}
}

// newInClusterClient returns a new Provider client that is expected to run
// inside the cluster.
func newInClusterClient(endpoint string) (*clientImpl, error) {
	config, err := rest.InClusterConfig()
	if err != nil {
		return nil, fmt.Errorf("failed to create in-cluster configuration: %s", err)
	}

	if endpoint != "" {
		config.Host = endpoint
	}

	return createClientFromConfig(config)
}

// newExternalClusterClient returns a new Provider client that may run outside
// of the cluster.
// The endpoint parameter must not be empty.
func newExternalClusterClient(endpoint, token, caFilePath string) (*clientImpl, error) {
	if endpoint == "" {
		return nil, errors.New("endpoint missing for external cluster client")
	}

	config := &rest.Config{
		Host:        endpoint,
		BearerToken: token,
	}

	if caFilePath != "" {
		caData, err := ioutil.ReadFile(caFilePath)
		if err != nil {
			return nil, fmt.Errorf("failed to read CA file %s: %s", caFilePath, err)
		}

		config.TLSClientConfig = rest.TLSClientConfig{CAData: caData}
	}

	return createClientFromConfig(config)
}

func createClientFromConfig(c *rest.Config) (*clientImpl, error) {
	clientset, err := kubernetes.NewForConfig(c)
	if err != nil {
		return nil, err
	}

	return newClientImpl(clientset), nil
}

// WatchAll starts namespace-specific controllers for all relevant kinds.
func (c *clientImpl) WatchAll(namespaces Namespaces, stopCh <-chan struct{}) (<-chan interface{}, error) {
	eventCh := make(chan interface{}, 1)

	if len(namespaces) == 0 {
		namespaces = Namespaces{metav1.NamespaceAll}
		c.isNamespaceAll = true
	}

	eventHandler := c.newResourceEventHandler(eventCh)
	for _, ns := range namespaces {
		factory := informers.NewFilteredSharedInformerFactory(c.clientset, resyncPeriod, ns, nil)
		factory.Extensions().V1beta1().Ingresses().Informer().AddEventHandler(eventHandler)
		factory.Core().V1().Services().Informer().AddEventHandler(eventHandler)
		factory.Core().V1().Endpoints().Informer().AddEventHandler(eventHandler)
		c.factories[ns] = factory
	}

	for _, ns := range namespaces {
		c.factories[ns].Start(stopCh)
	}

	for _, ns := range namespaces {
		for t, ok := range c.factories[ns].WaitForCacheSync(stopCh) {
			if !ok {
				return nil, fmt.Errorf("timed out waiting for controller caches to sync %s in namespace %q", t.String(), ns)
			}
		}
	}

	// Do not wait for the Secrets store to get synced since we cannot rely on
	// users having granted RBAC permissions for this object.
	// https://github.com/containous/traefik/issues/1784 should improve the
	// situation here in the future.
	for _, ns := range namespaces {
		c.factories[ns].Core().V1().Secrets().Informer().AddEventHandler(eventHandler)
		c.factories[ns].Start(stopCh)
	}

	return eventCh, nil
}

// GetIngresses returns all Ingresses for observed namespaces in the cluster.
func (c *clientImpl) GetIngresses() []*extensionsv1beta1.Ingress {
	var result []*extensionsv1beta1.Ingress
	for ns, factory := range c.factories {
		ings, err := factory.Extensions().V1beta1().Ingresses().Lister().List(c.ingressLabelSelector)
		if err != nil {
			log.Errorf("Failed to list ingresses in namespace %s: %s", ns, err)
		}
		for _, ing := range ings {
			result = append(result, ing)
		}
	}
	return result
}

// UpdateIngressStatus updates an Ingress with a provided status.
func (c *clientImpl) UpdateIngressStatus(namespace, name, ip, hostname string) error {
	keyName := namespace + "/" + name

	item, exists, err := c.factories[c.lookupNamespace(namespace)].Extensions().V1beta1().Ingresses().Informer().GetStore().GetByKey(keyName)
	if err != nil {
		return fmt.Errorf("failed to get ingress %s with error: %v", keyName, err)
	}
	if !exists {
		return fmt.Errorf("failed to update ingress %s because it does not exist", keyName)
	}

	ing := item.(*extensionsv1beta1.Ingress)
	if ing.Status.LoadBalancer.Ingress[0].Hostname == hostname && ing.Status.LoadBalancer.Ingress[0].IP == ip {
		// If status is already set, skip update
		log.Debugf("Skipping status update on ingress %s/%s", ing.Namespace, ing.Name)
	} else {
		ingCopy := ing.DeepCopy()
		ingCopy.Status = extensionsv1beta1.IngressStatus{LoadBalancer: corev1.LoadBalancerStatus{Ingress: []corev1.LoadBalancerIngress{{IP: ip, Hostname: hostname}}}}

		_, err := c.clientset.ExtensionsV1beta1().Ingresses(ingCopy.Namespace).UpdateStatus(ingCopy)
		if err != nil {
			return fmt.Errorf("failed to update ingress status %s with error: %v", keyName, err)
		}
		log.Infof("Updated status on ingress %s", keyName)
	}
	return nil
}

// GetService returns the named service from the given namespace.
func (c *clientImpl) GetService(namespace, name string) (*corev1.Service, bool, error) {
	var service *corev1.Service
	item, exists, err := c.factories[c.lookupNamespace(namespace)].Core().V1().Services().Informer().GetStore().GetByKey(namespace + "/" + name)
	if item != nil {
		service = item.(*corev1.Service)
	}
	return service, exists, err
}

// GetEndpoints returns the named endpoints from the given namespace.
func (c *clientImpl) GetEndpoints(namespace, name string) (*corev1.Endpoints, bool, error) {
	var endpoint *corev1.Endpoints
	item, exists, err := c.factories[c.lookupNamespace(namespace)].Core().V1().Endpoints().Informer().GetStore().GetByKey(namespace + "/" + name)
	if item != nil {
		endpoint = item.(*corev1.Endpoints)
	}
	return endpoint, exists, err
}

// GetSecret returns the named secret from the given namespace.
func (c *clientImpl) GetSecret(namespace, name string) (*corev1.Secret, bool, error) {
	var secret *corev1.Secret
	item, exists, err := c.factories[c.lookupNamespace(namespace)].Core().V1().Secrets().Informer().GetStore().GetByKey(namespace + "/" + name)
	if err == nil && item != nil {
		secret = item.(*corev1.Secret)
	}
	return secret, exists, err
}

// lookupNamespace returns the lookup namespace key for the given namespace.
// When listening on all namespaces, it returns the client-go identifier ("")
// for all-namespaces. Otherwise, it returns the given namespace.
// The distinction is necessary because we index all informers on the special
// identifier iff all-namespaces are requested but receive specific namespace
// identifiers from the Kubernetes API, so we have to bridge this gap.
func (c *clientImpl) lookupNamespace(ns string) string {
	if c.isNamespaceAll {
		return metav1.NamespaceAll
	}
	return ns
}

func (c *clientImpl) newResourceEventHandler(events chan<- interface{}) cache.ResourceEventHandler {
	return &cache.FilteringResourceEventHandler{
		FilterFunc: func(obj interface{}) bool {
			// Ignore Ingresses that do not match our custom label selector.
			if ing, ok := obj.(*extensionsv1beta1.Ingress); ok {
				lbls := labels.Set(ing.GetLabels())
				return c.ingressLabelSelector.Matches(lbls)
			}
			return true
		},
		Handler: &resourceEventHandler{events},
	}
}

// eventHandlerFunc will pass the obj on to the events channel or drop it.
// This is so passing the events along won't block in the case of high volume.
// The events are only used for signalling anyway so dropping a few is ok.
func eventHandlerFunc(events chan<- interface{}, obj interface{}) {
	select {
	case events <- obj:
	default:
	}
}
<commit_msg>Fix panic setting ingress status<commit_after>package kubernetes

import (
	"errors"
	"fmt"
	"io/ioutil"
	"time"

	"github.com/containous/traefik/log"
	corev1 "k8s.io/api/core/v1"
	extensionsv1beta1 "k8s.io/api/extensions/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"
)

const resyncPeriod = 10 * time.Minute

type resourceEventHandler struct {
	ev chan<- interface{}
}

func (reh *resourceEventHandler) OnAdd(obj interface{}) {
	eventHandlerFunc(reh.ev, obj)
}

func (reh *resourceEventHandler) OnUpdate(oldObj, newObj interface{}) {
	eventHandlerFunc(reh.ev, newObj)
}

func (reh *resourceEventHandler) OnDelete(obj interface{}) {
	eventHandlerFunc(reh.ev, obj)
}

// Client is a client for the Provider master.
// WatchAll starts the watch of the Provider resources and updates the stores.
// The stores can then be accessed via the Get* functions.
type Client interface {
	WatchAll(namespaces Namespaces, stopCh <-chan struct{}) (<-chan interface{}, error)
	GetIngresses() []*extensionsv1beta1.Ingress
	GetService(namespace, name string) (*corev1.Service, bool, error)
	GetSecret(namespace, name string) (*corev1.Secret, bool, error)
	GetEndpoints(namespace, name string) (*corev1.Endpoints, bool, error)
	UpdateIngressStatus(namespace, name, ip, hostname string) error
}

type clientImpl struct {
	clientset            *kubernetes.Clientset
	factories            map[string]informers.SharedInformerFactory
	ingressLabelSelector labels.Selector
	isNamespaceAll       bool
}

func newClientImpl(clientset *kubernetes.Clientset) *clientImpl {
	return &clientImpl{
		clientset: clientset,
		factories: make(map[string]informers.SharedInformerFactory),
	}
}

// newInClusterClient returns a new Provider client that is expected to run
// inside the cluster.
func newInClusterClient(endpoint string) (*clientImpl, error) {
	config, err := rest.InClusterConfig()
	if err != nil {
		return nil, fmt.Errorf("failed to create in-cluster configuration: %s", err)
	}

	if endpoint != "" {
		config.Host = endpoint
	}

	return createClientFromConfig(config)
}

// newExternalClusterClient returns a new Provider client that may run outside
// of the cluster.
// The endpoint parameter must not be empty.
func newExternalClusterClient(endpoint, token, caFilePath string) (*clientImpl, error) {
	if endpoint == "" {
		return nil, errors.New("endpoint missing for external cluster client")
	}

	config := &rest.Config{
		Host:        endpoint,
		BearerToken: token,
	}

	if caFilePath != "" {
		caData, err := ioutil.ReadFile(caFilePath)
		if err != nil {
			return nil, fmt.Errorf("failed to read CA file %s: %s", caFilePath, err)
		}

		config.TLSClientConfig = rest.TLSClientConfig{CAData: caData}
	}

	return createClientFromConfig(config)
}

func createClientFromConfig(c *rest.Config) (*clientImpl, error) {
	clientset, err := kubernetes.NewForConfig(c)
	if err != nil {
		return nil, err
	}

	return newClientImpl(clientset), nil
}

// WatchAll starts namespace-specific controllers for all relevant kinds.
func (c *clientImpl) WatchAll(namespaces Namespaces, stopCh <-chan struct{}) (<-chan interface{}, error) {
	eventCh := make(chan interface{}, 1)

	if len(namespaces) == 0 {
		namespaces = Namespaces{metav1.NamespaceAll}
		c.isNamespaceAll = true
	}

	eventHandler := c.newResourceEventHandler(eventCh)
	for _, ns := range namespaces {
		factory := informers.NewFilteredSharedInformerFactory(c.clientset, resyncPeriod, ns, nil)
		factory.Extensions().V1beta1().Ingresses().Informer().AddEventHandler(eventHandler)
		factory.Core().V1().Services().Informer().AddEventHandler(eventHandler)
		factory.Core().V1().Endpoints().Informer().AddEventHandler(eventHandler)
		c.factories[ns] = factory
	}

	for _, ns := range namespaces {
		c.factories[ns].Start(stopCh)
	}

	for _, ns := range namespaces {
		for t, ok := range c.factories[ns].WaitForCacheSync(stopCh) {
			if !ok {
				return nil, fmt.Errorf("timed out waiting for controller caches to sync %s in namespace %q", t.String(), ns)
			}
		}
	}

	// Do not wait for the Secrets store to get synced since we cannot rely on
	// users having granted RBAC permissions for this object.
	// https://github.com/containous/traefik/issues/1784 should improve the
	// situation here in the future.
	for _, ns := range namespaces {
		c.factories[ns].Core().V1().Secrets().Informer().AddEventHandler(eventHandler)
		c.factories[ns].Start(stopCh)
	}

	return eventCh, nil
}

// GetIngresses returns all Ingresses for observed namespaces in the cluster.
func (c *clientImpl) GetIngresses() []*extensionsv1beta1.Ingress {
	var result []*extensionsv1beta1.Ingress
	for ns, factory := range c.factories {
		ings, err := factory.Extensions().V1beta1().Ingresses().Lister().List(c.ingressLabelSelector)
		if err != nil {
			log.Errorf("Failed to list ingresses in namespace %s: %s", ns, err)
		}
		for _, ing := range ings {
			result = append(result, ing)
		}
	}
	return result
}

// UpdateIngressStatus updates an Ingress with a provided status.
func (c *clientImpl) UpdateIngressStatus(namespace, name, ip, hostname string) error {
	keyName := namespace + "/" + name

	item, exists, err := c.factories[c.lookupNamespace(namespace)].Extensions().V1beta1().Ingresses().Informer().GetStore().GetByKey(keyName)
	if err != nil {
		return fmt.Errorf("failed to get ingress %s with error: %v", keyName, err)
	}
	if !exists {
		return fmt.Errorf("failed to update ingress %s because it does not exist", keyName)
	}

	ing := item.(*extensionsv1beta1.Ingress)
	if len(ing.Status.LoadBalancer.Ingress) > 0 {
		if ing.Status.LoadBalancer.Ingress[0].Hostname == hostname && ing.Status.LoadBalancer.Ingress[0].IP == ip {
			// If status is already set, skip update
			log.Debugf("Skipping status update on ingress %s/%s", ing.Namespace, ing.Name)
			return nil
		}
	}
	ingCopy := ing.DeepCopy()
	ingCopy.Status = extensionsv1beta1.IngressStatus{LoadBalancer: corev1.LoadBalancerStatus{Ingress: []corev1.LoadBalancerIngress{{IP: ip, Hostname: hostname}}}}

	_, err = c.clientset.ExtensionsV1beta1().Ingresses(ingCopy.Namespace).UpdateStatus(ingCopy)
	if err != nil {
		return fmt.Errorf("failed to update ingress status %s with error: %v", keyName, err)
	}
	log.Infof("Updated status on ingress %s", keyName)
	return nil
}

// GetService returns the named service from the given namespace.
func (c *clientImpl) GetService(namespace, name string) (*corev1.Service, bool, error) {
	var service *corev1.Service
	item, exists, err := c.factories[c.lookupNamespace(namespace)].Core().V1().Services().Informer().GetStore().GetByKey(namespace + "/" + name)
	if item != nil {
		service = item.(*corev1.Service)
	}
	return service, exists, err
}

// GetEndpoints returns the named endpoints from the given namespace.
func (c *clientImpl) GetEndpoints(namespace, name string) (*corev1.Endpoints, bool, error) {
	var endpoint *corev1.Endpoints
	item, exists, err := c.factories[c.lookupNamespace(namespace)].Core().V1().Endpoints().Informer().GetStore().GetByKey(namespace + "/" + name)
	if item != nil {
		endpoint = item.(*corev1.Endpoints)
	}
	return endpoint, exists, err
}

// GetSecret returns the named secret from the given namespace.
func (c *clientImpl) GetSecret(namespace, name string) (*corev1.Secret, bool, error) {
	var secret *corev1.Secret
	item, exists, err := c.factories[c.lookupNamespace(namespace)].Core().V1().Secrets().Informer().GetStore().GetByKey(namespace + "/" + name)
	if err == nil && item != nil {
		secret = item.(*corev1.Secret)
	}
	return secret, exists, err
}

// lookupNamespace returns the lookup namespace key for the given namespace.
// When listening on all namespaces, it returns the client-go identifier ("")
// for all-namespaces. Otherwise, it returns the given namespace.
// The distinction is necessary because we index all informers on the special
// identifier iff all-namespaces are requested but receive specific namespace
// identifiers from the Kubernetes API, so we have to bridge this gap.
func (c *clientImpl) lookupNamespace(ns string) string {
	if c.isNamespaceAll {
		return metav1.NamespaceAll
	}
	return ns
}

func (c *clientImpl) newResourceEventHandler(events chan<- interface{}) cache.ResourceEventHandler {
	return &cache.FilteringResourceEventHandler{
		FilterFunc: func(obj interface{}) bool {
			// Ignore Ingresses that do not match our custom label selector.
			if ing, ok := obj.(*extensionsv1beta1.Ingress); ok {
				lbls := labels.Set(ing.GetLabels())
				return c.ingressLabelSelector.Matches(lbls)
			}
			return true
		},
		Handler: &resourceEventHandler{events},
	}
}

// eventHandlerFunc will pass the obj on to the events channel or drop it.
// This is so passing the events along won't block in the case of high volume.
// The events are only used for signalling anyway so dropping a few is ok.
func eventHandlerFunc(events chan<- interface{}, obj interface{}) {
	select {
	case events <- obj:
	default:
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"errors"
	"fmt"
	"math"
	"strconv"
	"sync"

	"github.com/fsouza/go-dockerclient"
	"github.com/tsuru/config"
	"github.com/tsuru/docker-cluster/cluster"
	"github.com/tsuru/tsuru/app"
	"github.com/tsuru/tsuru/log"
	"github.com/tsuru/tsuru/net"
	"github.com/tsuru/tsuru/provision/docker/container"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

// errNoDefaultPool is the error returned when no default hosts are configured in
// the segregated scheduler.
var errNoDefaultPool = errors.New("no default pool configured in the scheduler: you should create a default pool.")

type segregatedScheduler struct {
	hostMutex           sync.Mutex
	maxMemoryRatio      float32
	TotalMemoryMetadata string
	provisioner         *dockerProvisioner
	// ignored containers is only set in provisioner returned by
	// cloneProvisioner which will set this field to exclude some container
	// ids from balancing (containers being removed by rebalance usually).
	ignoredContainers []string
}

func (s *segregatedScheduler) Schedule(c *cluster.Cluster, opts docker.CreateContainerOptions, schedulerOpts cluster.SchedulerOptions) (cluster.Node, error) {
	schedOpts, ok := schedulerOpts.(*container.SchedulerOpts)
	if !ok {
		return cluster.Node{}, &container.SchedulerError{
			Base: fmt.Errorf("invalid scheduler opts: %#v", schedulerOpts),
		}
	}
	a, _ := app.GetByName(schedOpts.AppName)
	nodes, err := s.provisioner.Nodes(a)
	if err != nil {
		return cluster.Node{}, &container.SchedulerError{Base: err}
	}
	nodes, err = s.filterByMemoryUsage(a, nodes, s.maxMemoryRatio, s.TotalMemoryMetadata)
	if err != nil {
		return cluster.Node{}, &container.SchedulerError{Base: err}
	}
	node, err := s.chooseNodeToAdd(nodes, opts.Name, schedOpts.AppName, schedOpts.ProcessName)
	if err != nil {
		return cluster.Node{}, &container.SchedulerError{Base: err}
	}
	if schedOpts.ActionLimiter != nil {
		schedOpts.LimiterDone = schedOpts.ActionLimiter.Start(net.URLToHost(node))
	}
	return cluster.Node{Address: node}, nil
}

func (s *segregatedScheduler) filterByMemoryUsage(a *app.App, nodes []cluster.Node, maxMemoryRatio float32, TotalMemoryMetadata string) ([]cluster.Node, error) {
	if maxMemoryRatio == 0 || TotalMemoryMetadata == "" {
		return nodes, nil
	}
	hosts := make([]string, len(nodes))
	for i := range nodes {
		hosts[i] = net.URLToHost(nodes[i].Address)
	}
	containers, err := s.provisioner.ListContainers(bson.M{"hostaddr": bson.M{"$in": hosts}, "id": bson.M{"$nin": s.ignoredContainers}})
	if err != nil {
		return nil, err
	}
	hostReserved := make(map[string]int64)
	for _, cont := range containers {
		contApp, err := app.GetByName(cont.AppName)
		if err != nil {
			return nil, err
		}
		hostReserved[cont.HostAddr] += contApp.Plan.Memory
	}
	megabyte := float64(1024 * 1024)
	nodeList := make([]cluster.Node, 0, len(nodes))
	for _, node := range nodes {
		totalMemory, _ := strconv.ParseFloat(node.Metadata[TotalMemoryMetadata], 64)
		shouldAdd := true
		if totalMemory != 0 {
			maxMemory := totalMemory * float64(maxMemoryRatio)
			host := net.URLToHost(node.Address)
			nodeReserved := hostReserved[host] + a.Plan.Memory
			if nodeReserved > int64(maxMemory) {
				shouldAdd = false
				tryingToReserveMB := float64(a.Plan.Memory) / megabyte
				reservedMB := float64(hostReserved[host]) / megabyte
				limitMB := maxMemory / megabyte
				log.Errorf("Node %q has reached its memory limit. "+
					"Limit %0.4fMB. Reserved: %0.4fMB. Needed additional %0.4fMB",
					host, limitMB, reservedMB, tryingToReserveMB)
			}
		}
		if shouldAdd {
			nodeList = append(nodeList, node)
		}
	}
	if len(nodeList) == 0 {
		autoScaleEnabled, _ := config.GetBool("docker:auto-scale:enabled")
		errMsg := fmt.Sprintf("no nodes found with enough memory for container of %q: %0.4fMB",
			a.Name, float64(a.Plan.Memory)/megabyte)
		if autoScaleEnabled {
			// Allow going over quota temporarily because auto-scale will be
			// able to detect this and automatically add a new nodes.
			log.Errorf("WARNING: %s. Will ignore memory restrictions.", errMsg)
			return nodes, nil
		}
		return nil, errors.New(errMsg)
	}
	return nodeList, nil
}

type nodeAggregate struct {
	HostAddr string `bson:"_id"`
	Count    int
}

// aggregateContainersBy aggregates and counts how many containers
// exist each node that matches received filters
func (s *segregatedScheduler) aggregateContainersBy(matcher bson.M) (map[string]int, error) {
	coll := s.provisioner.Collection()
	defer coll.Close()
	pipe := coll.Pipe([]bson.M{
		matcher,
		{"$group": bson.M{"_id": "$hostaddr", "count": bson.M{"$sum": 1}}},
	})
	var results []nodeAggregate
	err := pipe.All(&results)
	if err != nil {
		return nil, err
	}
	countMap := make(map[string]int)
	for _, result := range results {
		countMap[result.HostAddr] = result.Count
	}
	return countMap, nil
}

func (s *segregatedScheduler) aggregateContainersByHost(hosts []string) (map[string]int, error) {
	return s.aggregateContainersBy(bson.M{"$match": bson.M{"hostaddr": bson.M{"$in": hosts}, "id": bson.M{"$nin": s.ignoredContainers}}})
}

func (s *segregatedScheduler) aggregateContainersByHostAppProcess(hosts []string, appName, process string) (map[string]int, error) {
	matcher := bson.M{
		"appname":  appName,
		"hostaddr": bson.M{"$in": hosts},
		"id":       bson.M{"$nin": s.ignoredContainers},
	}
	if process == "" {
		matcher["$or"] = []bson.M{{"processname": bson.M{"$exists": false}}, {"processname": ""}}
	} else {
		matcher["processname"] = process
	}
	return s.aggregateContainersBy(bson.M{"$match": matcher})
}

func (s *segregatedScheduler) GetRemovableContainer(appName string, process string) (string, error) {
	a, _ := app.GetByName(appName)
	nodes, err := s.provisioner.Nodes(a)
	if err != nil {
		return "", err
	}
	return s.chooseContainerToRemove(nodes, appName, process)
}

type errContainerNotFound struct {
	AppName     string
	HostAddr    string
	ProcessName string
}

func (m *errContainerNotFound) Error() string {
	return fmt.Sprintf("Container of app %q with process %q was not found in server %q", m.AppName, m.ProcessName, m.HostAddr)
}

func (s *segregatedScheduler) getContainerFromHost(host string, appName, process string) (string, error) {
	coll := s.provisioner.Collection()
	defer coll.Close()
	var c container.Container
	query := bson.M{
		"id":       bson.M{"$nin": s.ignoredContainers},
		"appname":  appName,
		"hostaddr": net.URLToHost(host),
	}
	if process == "" {
		query["$or"] = []bson.M{{"processname": bson.M{"$exists": false}}, {"processname": ""}}
	} else {
		query["processname"] = process
	}
	err := coll.Find(query).Select(bson.M{"id": 1}).One(&c)
	if err == mgo.ErrNotFound {
		return "", &errContainerNotFound{AppName: appName, ProcessName: process, HostAddr: net.URLToHost(host)}
	}
	return c.ID, err
}

func (s *segregatedScheduler) nodesToHosts(nodes []cluster.Node) ([]string, map[string]string) {
	hosts := make([]string, len(nodes))
	hostsMap := make(map[string]string)
	// Only hostname is saved in the docker containers collection
	// so we need to extract and map then to the original node.
	for i, node := range nodes {
		host := net.URLToHost(node.Address)
		hosts[i] = host
		hostsMap[host] = node.Address
	}
	return hosts, hostsMap
}

// chooseNodeToAdd finds which is the node with the minimum number of containers
// and returns it
func (s *segregatedScheduler) chooseNodeToAdd(nodes []cluster.Node, contName string, appName, process string) (string, error) {
	log.Debugf("[scheduler] Possible nodes for container %s: %#v", contName, nodes)
	s.hostMutex.Lock()
	defer s.hostMutex.Unlock()
	chosenNode, _, err := s.minMaxNodes(nodes, appName, process)
	if err != nil {
		return "", err
	}
	log.Debugf("[scheduler] Chosen node for container %s: %#v", contName, chosenNode)
	if contName != "" {
		coll := s.provisioner.Collection()
		defer coll.Close()
		err = coll.Update(bson.M{"name": contName}, bson.M{"$set": bson.M{"hostaddr": net.URLToHost(chosenNode)}})
	}
	return chosenNode, err
}

// chooseContainerToRemove finds a container from the the node with maximum
// number of containers and returns it
func (s *segregatedScheduler) chooseContainerToRemove(nodes []cluster.Node, appName, process string) (string, error) {
	_, chosenNode, err := s.minMaxNodes(nodes, appName, process)
	if err != nil {
		return "", err
	}
	log.Debugf("[scheduler] Chosen node for remove a container: %#v", chosenNode)
	containerID, err := s.getContainerFromHost(chosenNode, appName, process)
	if err != nil {
		return "", err
	}
	return containerID, err
}

func appGroupCount(hostGroups map[string]int, appCountHost map[string]int) map[string]int {
	groupCounters := map[int]int{}
	for host, count := range appCountHost {
		groupCounters[hostGroups[host]] += count
	}
	result := map[string]int{}
	for host := range hostGroups {
		result[host] = groupCounters[hostGroups[host]]
	}
	return result
}

// Find the host with the minimum (good to add a new container) and maximum
// (good to remove a container) value for the pair [(number of containers for
// app-process), (number of containers in host)]
func (s *segregatedScheduler) minMaxNodes(nodes []cluster.Node, appName, process string) (string, string, error) {
	nodesPtr := make([]*cluster.Node, len(nodes))
	for i := range nodes {
		nodesPtr[i] = &nodes[i]
	}
	metaFreqList, _, err := splitMetadata(nodesPtr)
	if err != nil {
		log.Debugf("[scheduler] ignoring metadata diff when selecting node: %s", err)
	}
	hostGroupMap := map[string]int{}
	for i, m := range metaFreqList {
		for _, n := range m.nodes {
			hostGroupMap[net.URLToHost(n.Address)] = i
		}
	}
	hosts, hostsMap := s.nodesToHosts(nodes)
	hostCountMap, err := s.aggregateContainersByHost(hosts)
	if err != nil {
		return "", "", err
	}
	appCountMap, err := s.aggregateContainersByHostAppProcess(hosts, appName, process)
	if err != nil {
		return "", "", err
	}
	priorityEntries := []map[string]int{appGroupCount(hostGroupMap, appCountMap), appCountMap, hostCountMap}
	var minHost, maxHost string
	var minScore uint64 = math.MaxUint64
	var maxScore uint64 = 0
	for _, host := range hosts {
		var score uint64
		for i, e := range priorityEntries {
			score += uint64(e[host]) << uint((len(priorityEntries)-i-1)*(64/len(priorityEntries)))
		}
		if score < minScore {
			minScore = score
			minHost = host
		}
		if score > maxScore {
			maxScore = score
			maxHost = host
		}
	}
	return hostsMap[minHost], hostsMap[maxHost], nil
}
<commit_msg>provision/docker: remove unused code<commit_after>// Copyright 2016 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"errors"
	"fmt"
	"math"
	"strconv"
	"sync"

	"github.com/fsouza/go-dockerclient"
	"github.com/tsuru/config"
	"github.com/tsuru/docker-cluster/cluster"
	"github.com/tsuru/tsuru/app"
	"github.com/tsuru/tsuru/log"
	"github.com/tsuru/tsuru/net"
	"github.com/tsuru/tsuru/provision/docker/container"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

type segregatedScheduler struct {
	hostMutex           sync.Mutex
	maxMemoryRatio      float32
	TotalMemoryMetadata string
	provisioner         *dockerProvisioner
	// ignored containers is only set in provisioner returned by
	// cloneProvisioner which will set this field to exclude some container
	// ids from balancing (containers being removed by rebalance usually).
	ignoredContainers []string
}

func (s *segregatedScheduler) Schedule(c *cluster.Cluster, opts docker.CreateContainerOptions, schedulerOpts cluster.SchedulerOptions) (cluster.Node, error) {
	schedOpts, ok := schedulerOpts.(*container.SchedulerOpts)
	if !ok {
		return cluster.Node{}, &container.SchedulerError{
			Base: fmt.Errorf("invalid scheduler opts: %#v", schedulerOpts),
		}
	}
	a, _ := app.GetByName(schedOpts.AppName)
	nodes, err := s.provisioner.Nodes(a)
	if err != nil {
		return cluster.Node{}, &container.SchedulerError{Base: err}
	}
	nodes, err = s.filterByMemoryUsage(a, nodes, s.maxMemoryRatio, s.TotalMemoryMetadata)
	if err != nil {
		return cluster.Node{}, &container.SchedulerError{Base: err}
	}
	node, err := s.chooseNodeToAdd(nodes, opts.Name, schedOpts.AppName, schedOpts.ProcessName)
	if err != nil {
		return cluster.Node{}, &container.SchedulerError{Base: err}
	}
	if schedOpts.ActionLimiter != nil {
		schedOpts.LimiterDone = schedOpts.ActionLimiter.Start(net.URLToHost(node))
	}
	return cluster.Node{Address: node}, nil
}

func (s *segregatedScheduler) filterByMemoryUsage(a *app.App, nodes []cluster.Node, maxMemoryRatio float32, TotalMemoryMetadata string) ([]cluster.Node, error) {
	if maxMemoryRatio == 0 || TotalMemoryMetadata == "" {
		return nodes, nil
	}
	hosts := make([]string, len(nodes))
	for i := range nodes {
		hosts[i] = net.URLToHost(nodes[i].Address)
	}
	containers, err := s.provisioner.ListContainers(bson.M{"hostaddr": bson.M{"$in": hosts}, "id": bson.M{"$nin": s.ignoredContainers}})
	if err != nil {
		return nil, err
	}
	hostReserved := make(map[string]int64)
	for _, cont := range containers {
		contApp, err := app.GetByName(cont.AppName)
		if err != nil {
			return nil, err
		}
		hostReserved[cont.HostAddr] += contApp.Plan.Memory
	}
	megabyte := float64(1024 * 1024)
	nodeList := make([]cluster.Node, 0, len(nodes))
	for _, node := range nodes {
		totalMemory, _ := strconv.ParseFloat(node.Metadata[TotalMemoryMetadata], 64)
		shouldAdd := true
		if totalMemory != 0 {
			maxMemory := totalMemory * float64(maxMemoryRatio)
			host := net.URLToHost(node.Address)
			nodeReserved := hostReserved[host] + a.Plan.Memory
			if nodeReserved > int64(maxMemory) {
				shouldAdd = false
				tryingToReserveMB := float64(a.Plan.Memory) / megabyte
				reservedMB := float64(hostReserved[host]) / megabyte
				limitMB := maxMemory / megabyte
				log.Errorf("Node %q has reached its memory limit. "+
					"Limit %0.4fMB. Reserved: %0.4fMB. Needed additional %0.4fMB",
					host, limitMB, reservedMB, tryingToReserveMB)
			}
		}
		if shouldAdd {
			nodeList = append(nodeList, node)
		}
	}
	if len(nodeList) == 0 {
		autoScaleEnabled, _ := config.GetBool("docker:auto-scale:enabled")
		errMsg := fmt.Sprintf("no nodes found with enough memory for container of %q: %0.4fMB",
			a.Name, float64(a.Plan.Memory)/megabyte)
		if autoScaleEnabled {
			// Allow going over quota temporarily because auto-scale will be
			// able to detect this and automatically add a new nodes.
			log.Errorf("WARNING: %s. Will ignore memory restrictions.", errMsg)
			return nodes, nil
		}
		return nil, errors.New(errMsg)
	}
	return nodeList, nil
}

type nodeAggregate struct {
	HostAddr string `bson:"_id"`
	Count    int
}

// aggregateContainersBy aggregates and counts how many containers
// exist each node that matches received filters
func (s *segregatedScheduler) aggregateContainersBy(matcher bson.M) (map[string]int, error) {
	coll := s.provisioner.Collection()
	defer coll.Close()
	pipe := coll.Pipe([]bson.M{
		matcher,
		{"$group": bson.M{"_id": "$hostaddr", "count": bson.M{"$sum": 1}}},
	})
	var results []nodeAggregate
	err := pipe.All(&results)
	if err != nil {
		return nil, err
	}
	countMap := make(map[string]int)
	for _, result := range results {
		countMap[result.HostAddr] = result.Count
	}
	return countMap, nil
}

func (s *segregatedScheduler) aggregateContainersByHost(hosts []string) (map[string]int, error) {
	return s.aggregateContainersBy(bson.M{"$match": bson.M{"hostaddr": bson.M{"$in": hosts}, "id": bson.M{"$nin": s.ignoredContainers}}})
}

func (s *segregatedScheduler) aggregateContainersByHostAppProcess(hosts []string, appName, process string) (map[string]int, error) {
	matcher := bson.M{
		"appname":  appName,
		"hostaddr": bson.M{"$in": hosts},
		"id":       bson.M{"$nin": s.ignoredContainers},
	}
	if process == "" {
		matcher["$or"] = []bson.M{{"processname": bson.M{"$exists": false}}, {"processname": ""}}
	} else {
		matcher["processname"] = process
	}
	return s.aggregateContainersBy(bson.M{"$match": matcher})
}

func (s *segregatedScheduler) GetRemovableContainer(appName string, process string) (string, error) {
	a, _ := app.GetByName(appName)
	nodes, err := s.provisioner.Nodes(a)
	if err != nil {
		return "", err
	}
	return s.chooseContainerToRemove(nodes, appName, process)
}

type errContainerNotFound struct {
	AppName     string
	HostAddr    string
	ProcessName string
}

func (m *errContainerNotFound) Error() string {
	return fmt.Sprintf("Container of app %q with process %q was not found in server %q", m.AppName, m.ProcessName, m.HostAddr)
}

func (s *segregatedScheduler) getContainerFromHost(host string, appName, process string) (string, error) {
	coll := s.provisioner.Collection()
	defer coll.Close()
	var c container.Container
	query := bson.M{
		"id":       bson.M{"$nin": s.ignoredContainers},
		"appname":  appName,
		"hostaddr": net.URLToHost(host),
	}
	if process == "" {
		query["$or"] = []bson.M{{"processname": bson.M{"$exists": false}}, {"processname": ""}}
	} else {
		query["processname"] = process
	}
	err := coll.Find(query).Select(bson.M{"id": 1}).One(&c)
	if err == mgo.ErrNotFound {
		return "", &errContainerNotFound{AppName: appName, ProcessName: process, HostAddr: net.URLToHost(host)}
	}
	return c.ID, err
}

func (s *segregatedScheduler) nodesToHosts(nodes []cluster.Node) ([]string, map[string]string) {
	hosts := make([]string, len(nodes))
	hostsMap := make(map[string]string)
	// Only hostname is saved in the docker containers collection
	// so we need to extract and map then to the original node.
	for i, node := range nodes {
		host := net.URLToHost(node.Address)
		hosts[i] = host
		hostsMap[host] = node.Address
	}
	return hosts, hostsMap
}

// chooseNodeToAdd finds which is the node with the minimum number of containers
// and returns it
func (s *segregatedScheduler) chooseNodeToAdd(nodes []cluster.Node, contName string, appName, process string) (string, error) {
	log.Debugf("[scheduler] Possible nodes for container %s: %#v", contName, nodes)
	s.hostMutex.Lock()
	defer s.hostMutex.Unlock()
	chosenNode, _, err := s.minMaxNodes(nodes, appName, process)
	if err != nil {
		return "", err
	}
	log.Debugf("[scheduler] Chosen node for container %s: %#v", contName, chosenNode)
	if contName != "" {
		coll := s.provisioner.Collection()
		defer coll.Close()
		err = coll.Update(bson.M{"name": contName}, bson.M{"$set": bson.M{"hostaddr": net.URLToHost(chosenNode)}})
	}
	return chosenNode, err
}

// chooseContainerToRemove finds a container from the the node with maximum
// number of containers and returns it
func (s *segregatedScheduler) chooseContainerToRemove(nodes []cluster.Node, appName, process string) (string, error) {
	_, chosenNode, err := s.minMaxNodes(nodes, appName, process)
	if err != nil {
		return "", err
	}
	log.Debugf("[scheduler] Chosen node for remove a container: %#v", chosenNode)
	containerID, err := s.getContainerFromHost(chosenNode, appName, process)
	if err != nil {
		return "", err
	}
	return containerID, err
}

func appGroupCount(hostGroups map[string]int, appCountHost map[string]int) map[string]int {
	groupCounters := map[int]int{}
	for host, count := range appCountHost {
		groupCounters[hostGroups[host]] += count
	}
	result := map[string]int{}
	for host := range hostGroups {
		result[host] = groupCounters[hostGroups[host]]
	}
	return result
}

// Find the host with the minimum (good to add a new container) and maximum
// (good to remove a container) value for the pair [(number of containers for
// app-process), (number of containers in host)]
func (s *segregatedScheduler) minMaxNodes(nodes []cluster.Node, appName, process string) (string, string, error) {
	nodesPtr := make([]*cluster.Node, len(nodes))
	for i := range nodes {
		nodesPtr[i] = &nodes[i]
	}
	metaFreqList, _, err := splitMetadata(nodesPtr)
	if err != nil {
		log.Debugf("[scheduler] ignoring metadata diff when selecting node: %s", err)
	}
	hostGroupMap := map[string]int{}
	for i, m := range metaFreqList {
		for _, n := range m.nodes {
			hostGroupMap[net.URLToHost(n.Address)] = i
		}
	}
	hosts, hostsMap := s.nodesToHosts(nodes)
	hostCountMap, err := s.aggregateContainersByHost(hosts)
	if err != nil {
		return "", "", err
	}
	appCountMap, err := s.aggregateContainersByHostAppProcess(hosts, appName, process)
	if err != nil {
		return "", "", err
	}
	priorityEntries := []map[string]int{appGroupCount(hostGroupMap, appCountMap), appCountMap, hostCountMap}
	var minHost, maxHost string
	var minScore uint64 = math.MaxUint64
	var maxScore uint64 = 0
	for _, host := range hosts {
		var score uint64
		for i, e := range priorityEntries {
			score += uint64(e[host]) << uint((len(priorityEntries)-i-1)*(64/len(priorityEntries)))
		}
		if score < minScore {
			minScore = score
			minHost = host
		}
		if score > maxScore {
			maxScore = score
			maxHost = host
		}
	}
	return hostsMap[minHost], hostsMap[maxHost], nil
}
<|endoftext|>


<commit_before>package infrastructure

import (
	"context"
	"encoding/json"
	"fmt"
	"strings"
	"time"

	"github.com/webitel/cdr/src/conf"
	"github.com/webitel/cdr/src/entity"
	"github.com/webitel/cdr/src/logger"
	elastic "gopkg.in/olivere/elastic.v5"
)

type ElasticHandler struct {
	Client *elastic.Client
	Ctx    context.Context
}

var elasticConfig conf.Elastic

func NewElasticHandler() (*ElasticHandler, error) {
	elasticHandler := new(ElasticHandler)
	err := elasticHandler.Init()
	return elasticHandler, err
}

func (handler *ElasticHandler) Init() error {
	elasticConfig = conf.GetElastic()
	if !elasticConfig.Enable {
		return nil
	}
	var cdrTemplateMap string
	if bytes, err := json.Marshal(elasticConfig.CdrTemplate.Body); err == nil {
		cdrTemplateMap = string(bytes)
	} else {
		return err
	}
	var accountsTemplateMap string
	if bytes, err := json.Marshal(elasticConfig.AccountsTemplate.Body); err == nil {
		accountsTemplateMap = string(bytes)
	} else {
		return err
	}
	ctx := context.Background()
	for c := time.Tick(5 * time.Second); ; <-c {
		eClient, err := elastic.NewClient(elastic.SetURL(elasticConfig.Url), elastic.SetSniff(false))
		if err != nil {
			logger.Error(err.Error())
			continue
		}
		info, code, err := eClient.Ping(elasticConfig.Url).Do(ctx)
		if err != nil {
			logger.Error(err.Error())
			continue
		}
		handler.Client = eClient
		handler.Ctx = ctx
		logger.Debug("Elasticsearch returned with code %d and version %s", code, info.Version.Number)
		if err := handler.templatePrepare(elasticConfig.CdrTemplate, cdrTemplateMap); err != nil {
			logger.Error(err.Error())
			continue
		}
		if err := handler.templatePrepare(elasticConfig.AccountsTemplate, accountsTemplateMap); err != nil {
			logger.Error(err.Error())
			continue
		}
		break
	}
	return nil
}

func (handler *ElasticHandler) templatePrepare(template conf.ElasticTemplate, templateMap string) error {
	exists, err := handler.Client.IndexTemplateExists(template.Name).Do(handler.Ctx)
	if err != nil {
		return err
	}
	if !exists {
		// Create a new template.
		if err := handler.createTemplate(template.Name, templateMap); err != nil {
			return err
		}
	} else if elasticConfig.DeleteTemplate {
		deleteTemplate, err := handler.Client.IndexDeleteTemplate(template.Name).Name(template.Name).Do(handler.Ctx)
		if err != nil {
			return err
		}
		if !deleteTemplate.Acknowledged || deleteTemplate == nil {
			return fmt.Errorf("Template is not acknowledged")
		}
		if err := handler.createTemplate(template.Name, templateMap); err != nil {
			return err
		}
	}
	return nil
}

func (handler *ElasticHandler) createTemplate(templateName, templateMap string) error {
	createTemplate, err := handler.Client.IndexPutTemplate(templateName).Name(templateName).BodyString(templateMap).Do(handler.Ctx)
	if err != nil {
		return err
	}
	if !createTemplate.Acknowledged || createTemplate == nil {
		return fmt.Errorf("Template is not acknowledged")
		// Not acknowledged
	}
	logger.Debug("Elastic: put template: %s", templateName)
	return nil
}

func (handler *ElasticHandler) BulkInsert(calls []entity.ElasticCdr) (error, []entity.SqlCdr, []entity.SqlCdr) {
	bulkRequest := handler.Client.Bulk()
	for _, item := range calls {
		var tmpDomain string
		if item.DomainName != "" && !strings.ContainsAny(item.DomainName, ", & * & \\ & < & | & > & / & ?") {
			tmpDomain = "-" + item.DomainName
		}
		logger.DebugElastic("Elastic bulk item [Leg "+item.Leg+"]:", item.Uuid, item.DomainName)
		req := elastic.NewBulkUpdateRequest().Index(fmt.Sprintf("%s-%s-%v%v", elasticConfig.IndexNameCdr, strings.ToLower(item.Leg), time.Now().UTC().Year(), tmpDomain)).Type("cdr").RetryOnConflict(5).Id(item.Uuid).DocAsUpsert(true).Doc(item)
		bulkRequest = bulkRequest.Add(req)
	}
	res, err := bulkRequest.Do(handler.Ctx)
	if err != nil {
		return err, nil, nil
	}
	if res.Errors {
		var successCalls, errorCalls []entity.SqlCdr
		for _, item := range res.Items {
			if item["update"].Error != nil {
				errorCalls = append(errorCalls, entity.SqlCdr{Uuid: item["update"].Id})
				logger.ErrorElastic("Elastic [Leg A]", item["update"].Id, item["update"].Error.Type, item["update"].Index, item["update"].Error.Reason)
			} else {
				successCalls = append(successCalls, entity.SqlCdr{Uuid: item["update"].Id})
			}
		}
		return fmt.Errorf("Leg A: Bad response. Request has errors."), errorCalls, successCalls
	}
	return nil, nil, nil
}

func (handler *ElasticHandler) BulkStatus(accounts []entity.Account) (error, []entity.Account, []entity.Account) {
	bulkRequest := handler.Client.Bulk()
	for _, item := range accounts {
		var tmpDomain string
		if item.Domain != "" && !strings.ContainsAny(item.Domain, ", & * & \\ & < & | & > & / & ?") {
			tmpDomain = "-" + item.Domain
		}
		logger.DebugAccount("Elastic bulk item [Accounts]:", item.DisplayStatus, item.Account, item.Domain)
		req := elastic.NewBulkIndexRequest().Index(fmt.Sprintf("%s-%v%v", elasticConfig.IndexNameAccounts, time.Now().UTC().Year(), tmpDomain)).Type("accounts").RetryOnConflict(5).Id(item.Uuid).Doc(item)
		bulkRequest = bulkRequest.Add(req)
	}
	res, err := bulkRequest.Do(handler.Ctx)
	if err != nil {
		return err, nil, nil
	}
	if res.Errors {
		var successAcc, errorAcc []entity.Account
		for _, item := range res.Items {
			if item["update"].Error != nil {
				errorAcc = append(errorAcc, entity.Account{Uuid: item["update"].Id})
				logger.ErrorElastic("Elastic [Accounts]", item["update"].Id, item["update"].Error.Type, item["update"].Index, item["update"].Error.Reason)
			} else {
				successAcc = append(successAcc, entity.Account{Uuid: item["update"].Id})
			}
		}
		return fmt.Errorf("Accounts: Bad response. Request has errors."), errorAcc, successAcc
	}
	return nil, nil, nil
}

// func (handler *ElasticHandler) BulkUpdateLegs(calls []entity.ElasticCdr) (error, []entity.SqlCdr, []entity.SqlCdr) {
// 	bulkRequest := handler.Client.Bulk()
// 	for _, item := range calls {
// 		var tmpDomain string
// 		if item.DomainName != "" && !strings.ContainsAny(item.DomainName, ", & * & \\ & < & | & > & / & ?") {
// 			tmpDomain = "-" + item.DomainName
// 		}
// 		logger.DebugElastic("Elastic bulk item [Leg B]:", item.Uuid, item.DomainName)
// 		req := elastic.NewBulkUpdateRequest().Index(fmt.Sprintf("%s-%v%v", elasticConfig.IndexName, time.Now().UTC().Year(), tmpDomain)).Type(elasticConfig.TypeName).Id(item.Parent_uuid).RetryOnConflict(5).Upsert(map[string]interface{}{"legs_b": make([]bool, 0)}).ScriptedUpsert(true).Script(elastic.NewScriptInline("if(ctx._source.containsKey(\"legs_b\")){ctx._source.legs_b.add(params.v);}else{ctx._source.legs_b = new ArrayList(); ctx._source.legs_b.add(params.v);}").Lang("painless").Param("v", item))
// 		bulkRequest = bulkRequest.Add(req)
// 	}
// 	res, err := bulkRequest.Do(handler.Ctx)
// 	if err != nil {
// 		return err, nil, nil
// 	}
// 	if res.Errors {
// 		var successCalls, errorCalls []entity.SqlCdr
// 		for _, item := range res.Items {
// 			if item["update"].Error != nil {
// 				errorCalls = append(errorCalls, entity.SqlCdr{Uuid: item["update"].Id})
// 				logger.ErrorElastic("Elastic [Leg B]", item["update"].Id, item["update"].Error.Type, item["update"].Index, item["update"].Error.Reason)
// 			} else {
// 				successCalls = append(successCalls, entity.SqlCdr{Uuid: item["update"].Id})
// 			}
// 		}
// 		return fmt.Errorf("Leg B: Bad response. Request has errors."), errorCalls, successCalls
// 	}
// 	return nil, nil, nil
// }
<commit_msg>accounts check errors fix<commit_after>package infrastructure

import (
	"context"
	"encoding/json"
	"fmt"
	"strings"
	"time"

	"github.com/webitel/cdr/src/conf"
	"github.com/webitel/cdr/src/entity"
	"github.com/webitel/cdr/src/logger"
	elastic "gopkg.in/olivere/elastic.v5"
)

type ElasticHandler struct {
	Client *elastic.Client
	Ctx    context.Context
}

var elasticConfig conf.Elastic

func NewElasticHandler() (*ElasticHandler, error) {
	elasticHandler := new(ElasticHandler)
	err := elasticHandler.Init()
	return elasticHandler, err
}

func (handler *ElasticHandler) Init() error {
	elasticConfig = conf.GetElastic()
	if !elasticConfig.Enable {
		return nil
	}
	var cdrTemplateMap string
	if bytes, err := json.Marshal(elasticConfig.CdrTemplate.Body); err == nil {
		cdrTemplateMap = string(bytes)
	} else {
		return err
	}
	var accountsTemplateMap string
	if bytes, err := json.Marshal(elasticConfig.AccountsTemplate.Body); err == nil {
		accountsTemplateMap = string(bytes)
	} else {
		return err
	}
	ctx := context.Background()
	for c := time.Tick(5 * time.Second); ; <-c {
		eClient, err := elastic.NewClient(elastic.SetURL(elasticConfig.Url), elastic.SetSniff(false))
		if err != nil {
			logger.Error(err.Error())
			continue
		}
		info, code, err := eClient.Ping(elasticConfig.Url).Do(ctx)
		if err != nil {
			logger.Error(err.Error())
			continue
		}
		handler.Client = eClient
		handler.Ctx = ctx
		logger.Debug("Elasticsearch returned with code %d and version %s", code, info.Version.Number)
		if err := handler.templatePrepare(elasticConfig.CdrTemplate, cdrTemplateMap); err != nil {
			logger.Error(err.Error())
			continue
		}
		if err := handler.templatePrepare(elasticConfig.AccountsTemplate, accountsTemplateMap); err != nil {
			logger.Error(err.Error())
			continue
		}
		break
	}
	return nil
}

func (handler *ElasticHandler) templatePrepare(template conf.ElasticTemplate, templateMap string) error {
	exists, err := handler.Client.IndexTemplateExists(template.Name).Do(handler.Ctx)
	if err != nil {
		return err
	}
	if !exists {
		// Create a new template.
		if err := handler.createTemplate(template.Name, templateMap); err != nil {
			return err
		}
	} else if elasticConfig.DeleteTemplate {
		deleteTemplate, err := handler.Client.IndexDeleteTemplate(template.Name).Name(template.Name).Do(handler.Ctx)
		if err != nil {
			return err
		}
		if !deleteTemplate.Acknowledged || deleteTemplate == nil {
			return fmt.Errorf("Template is not acknowledged")
		}
		if err := handler.createTemplate(template.Name, templateMap); err != nil {
			return err
		}
	}
	return nil
}

func (handler *ElasticHandler) createTemplate(templateName, templateMap string) error {
	createTemplate, err := handler.Client.IndexPutTemplate(templateName).Name(templateName).BodyString(templateMap).Do(handler.Ctx)
	if err != nil {
		return err
	}
	if !createTemplate.Acknowledged || createTemplate == nil {
		return fmt.Errorf("Template is not acknowledged")
		// Not acknowledged
	}
	logger.Debug("Elastic: put template: %s", templateName)
	return nil
}

func (handler *ElasticHandler) BulkInsert(calls []entity.ElasticCdr) (error, []entity.SqlCdr, []entity.SqlCdr) {
	bulkRequest := handler.Client.Bulk()
	for _, item := range calls {
		var tmpDomain string
		if item.DomainName != "" && !strings.ContainsAny(item.DomainName, ", & * & \\ & < & | & > & / & ?") {
			tmpDomain = "-" + item.DomainName
		}
		logger.DebugElastic("Elastic bulk item [Leg "+item.Leg+"]:", item.Uuid, item.DomainName)
		req := elastic.NewBulkUpdateRequest().Index(fmt.Sprintf("%s-%s-%v%v", elasticConfig.IndexNameCdr, strings.ToLower(item.Leg), time.Now().UTC().Year(), tmpDomain)).Type("cdr").RetryOnConflict(5).Id(item.Uuid).DocAsUpsert(true).Doc(item)
		bulkRequest = bulkRequest.Add(req)
	}
	res, err := bulkRequest.Do(handler.Ctx)
	if err != nil {
		return err, nil, nil
	}
	if res.Errors {
		var successCalls, errorCalls []entity.SqlCdr
		for _, item := range res.Items {
			if item["update"].Error != nil {
				errorCalls = append(errorCalls, entity.SqlCdr{Uuid: item["update"].Id})
				logger.ErrorElastic("Elastic [Leg A]", item["update"].Id, item["update"].Error.Type, item["update"].Index, item["update"].Error.Reason)
			} else {
				successCalls = append(successCalls, entity.SqlCdr{Uuid: item["update"].Id})
			}
		}
		return fmt.Errorf("Leg A: Bad response. Request has errors."), errorCalls, successCalls
	}
	return nil, nil, nil
}

func (handler *ElasticHandler) BulkStatus(accounts []entity.Account) (error, []entity.Account, []entity.Account) {
	bulkRequest := handler.Client.Bulk()
	for _, item := range accounts {
		var tmpDomain string
		if item.Domain != "" && !strings.ContainsAny(item.Domain, ", & * & \\ & < & | & > & / & ?") {
			tmpDomain = "-" + item.Domain
		}
		logger.DebugAccount("Elastic bulk item [Accounts]:", item.DisplayStatus, item.Account, item.Domain)
		req := elastic.NewBulkIndexRequest().Index(fmt.Sprintf("%s-%v%v", elasticConfig.IndexNameAccounts, time.Now().UTC().Year(), tmpDomain)).Type("accounts").RetryOnConflict(5).Id(item.Uuid).Doc(item)
		bulkRequest = bulkRequest.Add(req)
	}
	res, err := bulkRequest.Do(handler.Ctx)
	if err != nil {
		return err, nil, nil
	}
	if res.Errors {
		var successAcc, errorAcc []entity.Account
		for _, item := range res.Items {
			if item["index"].Error != nil {
				errorAcc = append(errorAcc, entity.Account{Uuid: item["index"].Id})
				logger.ErrorElastic("Elastic [Accounts]", item["index"].Id, item["index"].Error.Type, item["index"].Index, item["index"].Error.Reason)
			} else {
				successAcc = append(successAcc, entity.Account{Uuid: item["index"].Id})
			}
		}
		return fmt.Errorf("Accounts: Bad response. Request has errors."), errorAcc, successAcc
	}
	return nil, nil, nil
}

// func (handler *ElasticHandler) BulkUpdateLegs(calls []entity.ElasticCdr) (error, []entity.SqlCdr, []entity.SqlCdr) {
// 	bulkRequest := handler.Client.Bulk()
// 	for _, item := range calls {
// 		var tmpDomain string
// 		if item.DomainName != "" && !strings.ContainsAny(item.DomainName, ", & * & \\ & < & | & > & / & ?") {
// 			tmpDomain = "-" + item.DomainName
// 		}
// 		logger.DebugElastic("Elastic bulk item [Leg B]:", item.Uuid, item.DomainName)
// 		req := elastic.NewBulkUpdateRequest().Index(fmt.Sprintf("%s-%v%v", elasticConfig.IndexName, time.Now().UTC().Year(), tmpDomain)).Type(elasticConfig.TypeName).Id(item.Parent_uuid).RetryOnConflict(5).Upsert(map[string]interface{}{"legs_b": make([]bool, 0)}).ScriptedUpsert(true).Script(elastic.NewScriptInline("if(ctx._source.containsKey(\"legs_b\")){ctx._source.legs_b.add(params.v);}else{ctx._source.legs_b = new ArrayList(); ctx._source.legs_b.add(params.v);}").Lang("painless").Param("v", item))
// 		bulkRequest = bulkRequest.Add(req)
// 	}
// 	res, err := bulkRequest.Do(handler.Ctx)
// 	if err != nil {
// 		return err, nil, nil
// 	}
// 	if res.Errors {
// 		var successCalls, errorCalls []entity.SqlCdr
// 		for _, item := range res.Items {
// 			if item["update"].Error != nil {
// 				errorCalls = append(errorCalls, entity.SqlCdr{Uuid: item["update"].Id})
// 				logger.ErrorElastic("Elastic [Leg B]", item["update"].Id, item["update"].Error.Type, item["update"].Index, item["update"].Error.Reason)
// 			} else {
// 				successCalls = append(successCalls, entity.SqlCdr{Uuid: item["update"].Id})
// 			}
// 		}
// 		return fmt.Errorf("Leg B: Bad response. Request has errors."), errorCalls, successCalls
// 	}
// 	return nil, nil, nil
// }
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package clone

import (
	"bytes"
	"fmt"
	"net/url"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"github.com/sirupsen/logrus"

	"k8s.io/apimachinery/pkg/util/sets"

	prowapi "k8s.io/test-infra/prow/apis/prowjobs/v1"
	"k8s.io/test-infra/prow/logrusutil"
)

// Run clones the refs under the prescribed directory and optionally
// configures the git username and email in the repository as well.
func Run(refs prowapi.Refs, dir, gitUserName, gitUserEmail, cookiePath string, env []string, oauthToken string) Record {
	if len(oauthToken) > 0 {
		logrus.SetFormatter(logrusutil.NewCensoringFormatter(logrus.StandardLogger().Formatter, func() sets.String {
			return sets.NewString(oauthToken)
		}))
	}
	logrus.WithFields(logrus.Fields{"refs": refs}).Info("Cloning refs")
	record := Record{Refs: refs}

	// This function runs the provided commands in order, logging them as they run,
	// aborting early and returning if any command fails.
	runCommands := func(commands []cloneCommand) error {
		for _, command := range commands {
			formattedCommand, output, err := command.run()
			logrus.WithFields(logrus.Fields{"command": formattedCommand, "output": output, "error": err}).Info("Ran command")
			message := ""
			if err != nil {
				message = err.Error()
				record.Failed = true
			}
			record.Commands = append(record.Commands, Command{Command: formattedCommand, Output: output, Error: message})
			if err != nil {
				return err
			}
		}
		return nil
	}

	g := gitCtxForRefs(refs, dir, env, oauthToken)
	if err := runCommands(g.commandsForBaseRef(refs, gitUserName, gitUserEmail, cookiePath)); err != nil {
		return record
	}

	timestamp, err := g.gitHeadTimestamp()
	if err != nil {
		timestamp = int(time.Now().Unix())
	}
	if err := runCommands(g.commandsForPullRefs(refs, timestamp)); err != nil {
		return record
	}

	finalSHA, err := g.gitRevParse()
	if err != nil {
		logrus.WithError(err).Warnf("Cannot resolve finalSHA for ref %#v", refs)
	} else {
		record.FinalSHA = finalSHA
	}

	return record
}

// PathForRefs determines the full path to where
// refs should be cloned
func PathForRefs(baseDir string, refs prowapi.Refs) string {
	var clonePath string
	if refs.PathAlias != "" {
		clonePath = refs.PathAlias
	} else {
		clonePath = fmt.Sprintf("github.com/%s/%s", refs.Org, refs.Repo)
	}

	// Support a baseDir of root "/"
	if baseDir == "/" {
		return fmt.Sprintf("%ssrc/%s", baseDir, clonePath)
	}
	return fmt.Sprintf("%s/src/%s", baseDir, clonePath)
}

// gitCtx collects a few common values needed for all git commands.
type gitCtx struct {
	cloneDir      string
	env           []string
	repositoryURI string
}

// gitCtxForRefs creates a gitCtx based on the provide refs and baseDir.
func gitCtxForRefs(refs prowapi.Refs, baseDir string, env []string, oauthToken string) gitCtx {
	g := gitCtx{
		cloneDir:      PathForRefs(baseDir, refs),
		env:           env,
		repositoryURI: fmt.Sprintf("https://github.com/%s/%s.git", refs.Org, refs.Repo),
	}
	if refs.CloneURI != "" {
		g.repositoryURI = refs.CloneURI
	}

	if len(oauthToken) > 0 {
		u, _ := url.Parse(g.repositoryURI)
		u.User = url.UserPassword(oauthToken, "x-oauth-basic")
		g.repositoryURI = u.String()
	}

	return g
}

func (g *gitCtx) gitCommand(args ...string) cloneCommand {
	return cloneCommand{dir: g.cloneDir, env: g.env, command: "git", args: args}
}

// commandsForBaseRef returns the list of commands needed to initialize and
// configure a local git directory, as well as fetch and check out the provided
// base ref.
func (g *gitCtx) commandsForBaseRef(refs prowapi.Refs, gitUserName, gitUserEmail, cookiePath string) []cloneCommand {
	commands := []cloneCommand{{dir: "/", env: g.env, command: "mkdir", args: []string{"-p", g.cloneDir}}}

	commands = append(commands, g.gitCommand("init"))
	if gitUserName != "" {
		commands = append(commands, g.gitCommand("config", "user.name", gitUserName))
	}
	if gitUserEmail != "" {
		commands = append(commands, g.gitCommand("config", "user.email", gitUserEmail))
	}
	if cookiePath != "" {
		commands = append(commands, g.gitCommand("config", "http.cookiefile", cookiePath))
	}

	if refs.CloneDepth > 0 {
		commands = append(commands, g.gitCommand("fetch", g.repositoryURI, "--tags", "--prune", "--depth", strconv.Itoa(refs.CloneDepth)))
		commands = append(commands, g.gitCommand("fetch", "--depth", strconv.Itoa(refs.CloneDepth), g.repositoryURI, refs.BaseRef))
	} else {
		commands = append(commands, g.gitCommand("fetch", g.repositoryURI, "--tags", "--prune"))
		commands = append(commands, g.gitCommand("fetch", g.repositoryURI, refs.BaseRef))
	}
	var target string
	if refs.BaseSHA != "" {
		target = refs.BaseSHA
	} else {
		target = "FETCH_HEAD"
	}
	// we need to be "on" the target branch after the sync
	// so we need to set the branch to point to the base ref,
	// but we cannot update a branch we are on, so in case we
	// are on the branch we are syncing, we check out the SHA
	// first and reset the branch second, then check out the
	// branch we just reset to be in the correct final state
	commands = append(commands, g.gitCommand("checkout", target))
	commands = append(commands, g.gitCommand("branch", "--force", refs.BaseRef, target))
	commands = append(commands, g.gitCommand("checkout", refs.BaseRef))

	return commands
}

// gitHeadTimestamp returns the timestamp of the HEAD commit as seconds from the
// UNIX epoch. If unable to read the timestamp for any reason (such as missing
// the git, or not using a git repo), it returns 0 and an error.
func (g *gitCtx) gitHeadTimestamp() (int, error) {
	gitShowCommand := g.gitCommand("show", "-s", "--format=format:%ct", "HEAD")
	_, gitOutput, err := gitShowCommand.run()
	if err != nil {
		logrus.WithError(err).Debug("Could not obtain timestamp of git HEAD")
		return 0, err
	}
	timestamp, convErr := strconv.Atoi(strings.TrimSpace(string(gitOutput)))
	if convErr != nil {
		logrus.WithError(convErr).Errorf("Failed to parse timestamp %q", gitOutput)
		return 0, convErr
	}
	return timestamp, nil
}

// gitTimestampEnvs returns the list of environment variables needed to override
// git's author and commit timestamps when creating new commits.
func gitTimestampEnvs(timestamp int) []string {
	return []string{
		fmt.Sprintf("GIT_AUTHOR_DATE=%d", timestamp),
		fmt.Sprintf("GIT_COMMITTER_DATE=%d", timestamp),
	}
}

// gitRevParse returns current commit from HEAD in a git tree
func (g *gitCtx) gitRevParse() (string, error) {
	gitRevParseCommand := g.gitCommand("rev-parse", "HEAD")
	_, commit, err := gitRevParseCommand.run()
	if err != nil {
		logrus.WithError(err).Error("git rev-parse HEAD failed!")
		return "", err
	}
	return strings.TrimSpace(commit), nil
}

// commandsForPullRefs returns the list of commands needed to fetch and
// merge any pull refs as well as submodules. These commands should be run only
// after the commands provided by commandsForBaseRef have been run
// successfully.
// Each merge commit will be created at sequential seconds after fakeTimestamp.
// It's recommended that fakeTimestamp be set to the timestamp of the base ref.
// This enables reproducible timestamps and git tree digests every time the same
// set of base and pull refs are used.
func (g *gitCtx) commandsForPullRefs(refs prowapi.Refs, fakeTimestamp int) []cloneCommand {
	var commands []cloneCommand
	for _, prRef := range refs.Pulls {
		ref := fmt.Sprintf("pull/%d/head", prRef.Number)
		if prRef.Ref != "" {
			ref = prRef.Ref
		}
		commands = append(commands, g.gitCommand("fetch", g.repositoryURI, ref))
		var prCheckout string
		if prRef.SHA != "" {
			prCheckout = prRef.SHA
		} else {
			prCheckout = "FETCH_HEAD"
		}
		fakeTimestamp++
		gitMergeCommand := g.gitCommand("merge", "--no-ff", prCheckout)
		gitMergeCommand.env = append(gitMergeCommand.env, gitTimestampEnvs(fakeTimestamp)...)
		commands = append(commands, gitMergeCommand)
	}

	// unless the user specifically asks us not to, init submodules
	if !refs.SkipSubmodules {
		commands = append(commands, g.gitCommand("submodule", "update", "--init", "--recursive"))
	}

	return commands
}

type cloneCommand struct {
	dir     string
	env     []string
	command string
	args    []string
}

func (c *cloneCommand) run() (string, string, error) {
	output := bytes.Buffer{}
	cmd := exec.Command(c.command, c.args...)
	cmd.Dir = c.dir
	cmd.Env = append(cmd.Env, c.env...)
	cmd.Stdout = &output
	cmd.Stderr = &output
	err := cmd.Run()
	return strings.Join(append([]string{c.command}, c.args...), " "), output.String(), err
}

func (c *cloneCommand) String() string {
	return fmt.Sprintf("PWD=%s %s %s %s", c.dir, strings.Join(c.env, " "), c.command, strings.Join(c.env, " "))
}
<commit_msg>Switched to Path.Join<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package clone

import (
	"bytes"
	"fmt"
	"net/url"
	"os/exec"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/sirupsen/logrus"

	"k8s.io/apimachinery/pkg/util/sets"

	prowapi "k8s.io/test-infra/prow/apis/prowjobs/v1"
	"k8s.io/test-infra/prow/logrusutil"
)

// Run clones the refs under the prescribed directory and optionally
// configures the git username and email in the repository as well.
func Run(refs prowapi.Refs, dir, gitUserName, gitUserEmail, cookiePath string, env []string, oauthToken string) Record {
	if len(oauthToken) > 0 {
		logrus.SetFormatter(logrusutil.NewCensoringFormatter(logrus.StandardLogger().Formatter, func() sets.String {
			return sets.NewString(oauthToken)
		}))
	}
	logrus.WithFields(logrus.Fields{"refs": refs}).Info("Cloning refs")
	record := Record{Refs: refs}

	// This function runs the provided commands in order, logging them as they run,
	// aborting early and returning if any command fails.
	runCommands := func(commands []cloneCommand) error {
		for _, command := range commands {
			formattedCommand, output, err := command.run()
			logrus.WithFields(logrus.Fields{"command": formattedCommand, "output": output, "error": err}).Info("Ran command")
			message := ""
			if err != nil {
				message = err.Error()
				record.Failed = true
			}
			record.Commands = append(record.Commands, Command{Command: formattedCommand, Output: output, Error: message})
			if err != nil {
				return err
			}
		}
		return nil
	}

	g := gitCtxForRefs(refs, dir, env, oauthToken)
	if err := runCommands(g.commandsForBaseRef(refs, gitUserName, gitUserEmail, cookiePath)); err != nil {
		return record
	}

	timestamp, err := g.gitHeadTimestamp()
	if err != nil {
		timestamp = int(time.Now().Unix())
	}
	if err := runCommands(g.commandsForPullRefs(refs, timestamp)); err != nil {
		return record
	}

	finalSHA, err := g.gitRevParse()
	if err != nil {
		logrus.WithError(err).Warnf("Cannot resolve finalSHA for ref %#v", refs)
	} else {
		record.FinalSHA = finalSHA
	}

	return record
}

// PathForRefs determines the full path to where
// refs should be cloned
func PathForRefs(baseDir string, refs prowapi.Refs) string {
	var clonePath string
	if refs.PathAlias != "" {
		clonePath = refs.PathAlias
	} else {
		clonePath = fmt.Sprintf("github.com/%s/%s", refs.Org, refs.Repo)
	}

	return path.Join(baseDir, "src", clonePath)
}

// gitCtx collects a few common values needed for all git commands.
type gitCtx struct {
	cloneDir      string
	env           []string
	repositoryURI string
}

// gitCtxForRefs creates a gitCtx based on the provide refs and baseDir.
func gitCtxForRefs(refs prowapi.Refs, baseDir string, env []string, oauthToken string) gitCtx {
	g := gitCtx{
		cloneDir:      PathForRefs(baseDir, refs),
		env:           env,
		repositoryURI: fmt.Sprintf("https://github.com/%s/%s.git", refs.Org, refs.Repo),
	}
	if refs.CloneURI != "" {
		g.repositoryURI = refs.CloneURI
	}

	if len(oauthToken) > 0 {
		u, _ := url.Parse(g.repositoryURI)
		u.User = url.UserPassword(oauthToken, "x-oauth-basic")
		g.repositoryURI = u.String()
	}

	return g
}

func (g *gitCtx) gitCommand(args ...string) cloneCommand {
	return cloneCommand{dir: g.cloneDir, env: g.env, command: "git", args: args}
}

// commandsForBaseRef returns the list of commands needed to initialize and
// configure a local git directory, as well as fetch and check out the provided
// base ref.
func (g *gitCtx) commandsForBaseRef(refs prowapi.Refs, gitUserName, gitUserEmail, cookiePath string) []cloneCommand {
	commands := []cloneCommand{{dir: "/", env: g.env, command: "mkdir", args: []string{"-p", g.cloneDir}}}

	commands = append(commands, g.gitCommand("init"))
	if gitUserName != "" {
		commands = append(commands, g.gitCommand("config", "user.name", gitUserName))
	}
	if gitUserEmail != "" {
		commands = append(commands, g.gitCommand("config", "user.email", gitUserEmail))
	}
	if cookiePath != "" {
		commands = append(commands, g.gitCommand("config", "http.cookiefile", cookiePath))
	}

	if refs.CloneDepth > 0 {
		commands = append(commands, g.gitCommand("fetch", g.repositoryURI, "--tags", "--prune", "--depth", strconv.Itoa(refs.CloneDepth)))
		commands = append(commands, g.gitCommand("fetch", "--depth", strconv.Itoa(refs.CloneDepth), g.repositoryURI, refs.BaseRef))
	} else {
		commands = append(commands, g.gitCommand("fetch", g.repositoryURI, "--tags", "--prune"))
		commands = append(commands, g.gitCommand("fetch", g.repositoryURI, refs.BaseRef))
	}
	var target string
	if refs.BaseSHA != "" {
		target = refs.BaseSHA
	} else {
		target = "FETCH_HEAD"
	}
	// we need to be "on" the target branch after the sync
	// so we need to set the branch to point to the base ref,
	// but we cannot update a branch we are on, so in case we
	// are on the branch we are syncing, we check out the SHA
	// first and reset the branch second, then check out the
	// branch we just reset to be in the correct final state
	commands = append(commands, g.gitCommand("checkout", target))
	commands = append(commands, g.gitCommand("branch", "--force", refs.BaseRef, target))
	commands = append(commands, g.gitCommand("checkout", refs.BaseRef))

	return commands
}

// gitHeadTimestamp returns the timestamp of the HEAD commit as seconds from the
// UNIX epoch. If unable to read the timestamp for any reason (such as missing
// the git, or not using a git repo), it returns 0 and an error.
func (g *gitCtx) gitHeadTimestamp() (int, error) {
	gitShowCommand := g.gitCommand("show", "-s", "--format=format:%ct", "HEAD")
	_, gitOutput, err := gitShowCommand.run()
	if err != nil {
		logrus.WithError(err).Debug("Could not obtain timestamp of git HEAD")
		return 0, err
	}
	timestamp, convErr := strconv.Atoi(strings.TrimSpace(string(gitOutput)))
	if convErr != nil {
		logrus.WithError(convErr).Errorf("Failed to parse timestamp %q", gitOutput)
		return 0, convErr
	}
	return timestamp, nil
}

// gitTimestampEnvs returns the list of environment variables needed to override
// git's author and commit timestamps when creating new commits.
func gitTimestampEnvs(timestamp int) []string {
	return []string{
		fmt.Sprintf("GIT_AUTHOR_DATE=%d", timestamp),
		fmt.Sprintf("GIT_COMMITTER_DATE=%d", timestamp),
	}
}

// gitRevParse returns current commit from HEAD in a git tree
func (g *gitCtx) gitRevParse() (string, error) {
	gitRevParseCommand := g.gitCommand("rev-parse", "HEAD")
	_, commit, err := gitRevParseCommand.run()
	if err != nil {
		logrus.WithError(err).Error("git rev-parse HEAD failed!")
		return "", err
	}
	return strings.TrimSpace(commit), nil
}

// commandsForPullRefs returns the list of commands needed to fetch and
// merge any pull refs as well as submodules. These commands should be run only
// after the commands provided by commandsForBaseRef have been run
// successfully.
// Each merge commit will be created at sequential seconds after fakeTimestamp.
// It's recommended that fakeTimestamp be set to the timestamp of the base ref.
// This enables reproducible timestamps and git tree digests every time the same
// set of base and pull refs are used.
func (g *gitCtx) commandsForPullRefs(refs prowapi.Refs, fakeTimestamp int) []cloneCommand {
	var commands []cloneCommand
	for _, prRef := range refs.Pulls {
		ref := fmt.Sprintf("pull/%d/head", prRef.Number)
		if prRef.Ref != "" {
			ref = prRef.Ref
		}
		commands = append(commands, g.gitCommand("fetch", g.repositoryURI, ref))
		var prCheckout string
		if prRef.SHA != "" {
			prCheckout = prRef.SHA
		} else {
			prCheckout = "FETCH_HEAD"
		}
		fakeTimestamp++
		gitMergeCommand := g.gitCommand("merge", "--no-ff", prCheckout)
		gitMergeCommand.env = append(gitMergeCommand.env, gitTimestampEnvs(fakeTimestamp)...)
		commands = append(commands, gitMergeCommand)
	}

	// unless the user specifically asks us not to, init submodules
	if !refs.SkipSubmodules {
		commands = append(commands, g.gitCommand("submodule", "update", "--init", "--recursive"))
	}

	return commands
}

type cloneCommand struct {
	dir     string
	env     []string
	command string
	args    []string
}

func (c *cloneCommand) run() (string, string, error) {
	output := bytes.Buffer{}
	cmd := exec.Command(c.command, c.args...)
	cmd.Dir = c.dir
	cmd.Env = append(cmd.Env, c.env...)
	cmd.Stdout = &output
	cmd.Stderr = &output
	err := cmd.Run()
	return strings.Join(append([]string{c.command}, c.args...), " "), output.String(), err
}

func (c *cloneCommand) String() string {
	return fmt.Sprintf("PWD=%s %s %s %s", c.dir, strings.Join(c.env, " "), c.command, strings.Join(c.env, " "))
}
<|endoftext|>


<commit_before>package ratelimiter

import (
	"sync"
	"testing"
	"time"
)

var testlimits = []int{1, 10, 50, 100, 1000}

func TestRateLimiterSingleThreaded(t *testing.T) {
	for i, limit := range testlimits {
		l := NewLimiter(limit)
		count := 0
		tick := time.NewTicker(time.Second)
		go func() {
			for range tick.C {
				// Allow a count up to one more than the limit as scheduling of
				// goroutine vs the main thread could cause this check to not be
				// run quite in time for limit.
				if count > limit+1 {
					t.Errorf("#%d: Too many operations per second. Expected %d, got %d", i, limit, count)
				}
				count = 0
			}
		}()

		for i := 0; i < 3*limit; i++ {
			l.Wait()
			count++
		}
		tick.Stop()
	}
}

func TestRateLimiterGoroutines(t *testing.T) {
	for i, limit := range testlimits {
		l := NewLimiter(limit)
		count := 0
		tick := time.NewTicker(time.Second)
		go func() {
			for range tick.C {
				// Allow a count up to one more than the limit as scheduling of
				// goroutine vs the main thread could cause this check to not be
				// run quite in time for limit.
				if count > limit+1 {
					t.Errorf("#%d: Too many operations per second. Expected %d, got %d", i, limit, count)
				}
				count = 0
			}
		}()

		var wg sync.WaitGroup
		for i := 0; i < 3*limit; i++ {
			wg.Add(1)
			go func() {
				l.Wait()
				count++
				wg.Done()
			}()
		}
		wg.Wait()
		tick.Stop()
	}
}
<commit_msg>go/fixchain: remove data races in unit test<commit_after>package ratelimiter

import (
	"sync"
	"sync/atomic"
	"testing"
	"time"
)

var testlimits = []int{1, 10, 50, 100, 1000}

func checkTicker(t *testing.T, tick *time.Ticker, count *int64, i, limit int) {
	for range tick.C {
		// Allow a count up to slightly more than the limit as scheduling of
		// goroutine vs the main thread could cause this check to not be
		// run quite in time for limit.
		allowed := int(float64(limit)*1.005) + 1
		v := atomic.LoadInt64(count)
		if v > int64(allowed) {
			t.Errorf("#%d: Too many operations per second. Expected ~%d, got %d", i, limit, v)
		}
		atomic.StoreInt64(count, 0)
	}
}

func TestRateLimiterSingleThreaded(t *testing.T) {
	for i, limit := range testlimits {
		l := NewLimiter(limit)
		count := int64(0)
		tick := time.NewTicker(time.Second)
		go checkTicker(t, tick, &count, i, limit)

		for i := 0; i < 3*limit; i++ {
			l.Wait()
			atomic.AddInt64(&count, 1)
		}
		tick.Stop()
	}
}

func TestRateLimiterGoroutines(t *testing.T) {
	for i, limit := range testlimits {
		l := NewLimiter(limit)
		count := int64(0)
		tick := time.NewTicker(time.Second)
		go checkTicker(t, tick, &count, i, limit)

		var wg sync.WaitGroup
		for i := 0; i < 3*limit; i++ {
			wg.Add(1)
			go func() {
				l.Wait()
				atomic.AddInt64(&count, 1)
				wg.Done()
			}()
		}
		wg.Wait()
		tick.Stop()
	}
}
<|endoftext|>


<commit_before>package models

import (
	"socialapi/workers/common/runner"
	"testing"

	"github.com/koding/bongo"
	. "github.com/smartystreets/goconvey/convey"
)

func TestAccountNewAccount(t *testing.T) {
	Convey("while testing new account", t, func() {
		Convey("Function call should return account", func() {
			So(NewAccount(), ShouldNotBeNil)
		})
	})
}

func TestAccountGetId(t *testing.T) {
	Convey("while testing get id", t, func() {
		Convey("Initialized struct ", func() {
			Convey("should return given id", func() {
				a := Account{Id: 42}
				So(a.GetId(), ShouldEqual, 42)
			})

			Convey("Uninitialized struct ", func() {
				Convey("should return 0", func() {
					So(NewAccount().GetId(), ShouldEqual, 0)
				})
				So(NewAccount(), ShouldNotBeNil)
			})
		})
	})
}

func TestAccountFetchOrCreate(t *testing.T) {
	r := runner.New("test")
	if err := r.Init(); err != nil {
		t.Fatalf("couldnt start bongo %s", err.Error())
	}
	defer r.Close()

	Convey("while fetching or creating account", t, func() {
		Convey("it should have old id", func() {
			a := NewAccount()
			err := a.FetchOrCreate()
			So(err, ShouldNotBeNil)
			So(err, ShouldEqual, ErrOldIdIsNotSet)
		})

		Convey("it should have error if nick contains guest-", func() {
			a := NewAccount()
			a.OldId = "oldestId"
			a.Nick = "guest-test"
			err := a.FetchOrCreate()
			So(err, ShouldNotBeNil)
			So(err, ShouldEqual, ErrGuestsAreNotAllowed)
		})

		Convey("it should not have error if required fields are exist", func() {
			a := NewAccount()
			a.OldId = "oldIdOfAccount"
			a.Nick = "WhatANick"
			err := a.FetchOrCreate()
			So(err, ShouldBeNil)
		})

	})
}

func TestAccountFetchAccountById(t *testing.T) {
	r := runner.New("test")
	if err := r.Init(); err != nil {
		t.Fatalf("couldnt start bongo %s", err.Error())
	}
	defer r.Close()

	Convey("while fetching an account by id", t, func() {
		Convey("it should not have error while fething", func() {
			// create account
			acc := createAccountWithTest()
			So(acc.Create(), ShouldBeNil)

			// fetch the account by id of account
			fa, err := FetchAccountById(acc.Id)
			// error should be nil
			// means that fetching is done successfully
			So(err, ShouldBeNil)
			// account in the db should equal to fetched account
			So(fa.Id, ShouldEqual, acc.Id)
			So(fa.OldId, ShouldEqual, acc.OldId)
			So(fa.Nick, ShouldEqual, acc.Nick)
		})

		Convey("it should have error if record is not found", func() {
			// init account
			a := NewAccount()
			a.Id = 12345
			a.OldId = "oldIdOfAccount"
			a.Nick = "WhatANick"

			// this account id is not exist
			_, err := FetchAccountById(a.Id)
			So(err, ShouldNotBeNil)
			So(err, ShouldEqual, bongo.RecordNotFound)

		})

	})
}

func TestAccountFetchOldIdsByAccountIds(t *testing.T) {
	r := runner.New("test")
	if err := r.Init(); err != nil {
		t.Fatalf("couldnt start bongo %s", err.Error())
	}
	defer r.Close()

	Convey("while fetching Old Ids by account ids", t, func() {
		Convey("it should have account id (ids of length) more than zero", func() {
			acc := []int64{}
			foi, err := FetchOldIdsByAccountIds(acc)
			So(err, ShouldBeNil)
			So(foi, ShouldBeEmpty)
		})

		Convey("it should not have error if account is exist in db", func() {
			// create account
			acc := createAccountWithTest()
			So(acc.Create(), ShouldBeNil)
			// we created slice to send to FetchOldIdsByAccountIds as argument
			idd := []int64{acc.Id}

			// FetchOldIdsByAccountIds returns as slice
			foi, err := FetchOldIdsByAccountIds(idd)
			So(err, ShouldBeNil)
			// used shouldcontain because foi is a slice
			So(foi, ShouldContain, acc.OldId)

		})

		Convey("it should append successfully", func() {
			acc1 := NewAccount()
			acc1.Id = 1
			acc1.OldId = "11"
			acc1.Nick = "acc1"
			So(acc1.Create(), ShouldBeNil)

			acc2 := NewAccount()
			acc2.Id = 2
			acc2.OldId = "22"
			acc2.Nick = "acc2"
			So(acc2.Create(), ShouldBeNil)

			idd := []int64{acc1.Id, acc2.Id}
			old := []string{acc1.OldId, acc2.OldId}

			foi, err := FetchOldIdsByAccountIds(idd)
			So(err, ShouldBeNil)
			So(foi[1], ShouldEqual, old[1])

		})

	})
}

func TestAccountCreateFollowingFeedChannel(t *testing.T) {
	r := runner.New("test")
	if err := r.Init(); err != nil {
		t.Fatalf("couldnt start bongo %s", err.Error())
	}
	defer r.Close()

	Convey("while creating feed channel", t, func() {
		Convey("it should have account id", func() {
			// create account
			acc := NewAccount()

			_, err := acc.CreateFollowingFeedChannel()
			So(err, ShouldNotBeNil)
			So(err, ShouldEqual, ErrAccountIdIsNotSet)
		})

		Convey("it should have creator id as account id", func() {
			// create account
			acc := createAccountWithTest()
			So(acc.Create(), ShouldBeNil)

			cff, err := acc.CreateFollowingFeedChannel()
			So(err, ShouldBeNil)
			So(cff.CreatorId, ShouldEqual, acc.Id)
		})

		Convey("it should have channel name as required", func() {
			// create account
			acc := NewAccount()
			acc.Id = 1
			acc.OldId = "11"
			acc.Nick = "acc1"
			So(acc.Create(), ShouldBeNil)

			cff, err := acc.CreateFollowingFeedChannel()
			So(err, ShouldBeNil)
			So(cff.Name, ShouldEqual, "1-FollowingFeedChannel")
		})

		Convey("it should have group name as Channel_KODING_NAME", func() {
			// create account
			acc := createAccountWithTest()
			So(acc.Create(), ShouldBeNil)

			cff, err := acc.CreateFollowingFeedChannel()
			So(err, ShouldBeNil)
			So(cff.GroupName, ShouldEqual, Channel_KODING_NAME)
		})

		Convey("it should have purpose as Following Feed for me", func() {
			// create account
			acc := createAccountWithTest()
			So(acc.Create(), ShouldBeNil)

			cff, err := acc.CreateFollowingFeedChannel()
			So(err, ShouldBeNil)
			So(cff.Purpose, ShouldEqual, "Following Feed for Me")
		})

		Convey("it should have type constant as Channel_TYPE_FOLLOWERS", func() {
			// create account
			acc := createAccountWithTest()
			So(acc.Create(), ShouldBeNil)

			cff, err := acc.CreateFollowingFeedChannel()
			So(err, ShouldBeNil)
			So(cff.TypeConstant, ShouldEqual, Channel_TYPE_FOLLOWERS)
		})

	})
}

func TestAccountUnMarkAsTroll(t *testing.T) {
	r := runner.New("test")
	if err := r.Init(); err != nil {
		t.Fatalf("couldnt start bongo %s", err.Error())
	}
	defer r.Close()

	Convey("while unmarking as troll", t, func() {
		Convey("it should have account id", func() {
			acc := NewAccount()

			err := acc.UnMarkAsTroll()
			So(err, ShouldNotBeNil)
			So(err, ShouldEqual, ErrAccountIdIsNotSet)
		})
	})
}
<commit_msg>social/tests: unmarkAsTroll test is added for recordNotFound, in account<commit_after>package models

import (
	"socialapi/workers/common/runner"
	"testing"

	"github.com/koding/bongo"
	. "github.com/smartystreets/goconvey/convey"
)

func TestAccountNewAccount(t *testing.T) {
	Convey("while testing new account", t, func() {
		Convey("Function call should return account", func() {
			So(NewAccount(), ShouldNotBeNil)
		})
	})
}

func TestAccountGetId(t *testing.T) {
	Convey("while testing get id", t, func() {
		Convey("Initialized struct ", func() {
			Convey("should return given id", func() {
				a := Account{Id: 42}
				So(a.GetId(), ShouldEqual, 42)
			})

			Convey("Uninitialized struct ", func() {
				Convey("should return 0", func() {
					So(NewAccount().GetId(), ShouldEqual, 0)
				})
				So(NewAccount(), ShouldNotBeNil)
			})
		})
	})
}

func TestAccountFetchOrCreate(t *testing.T) {
	r := runner.New("test")
	if err := r.Init(); err != nil {
		t.Fatalf("couldnt start bongo %s", err.Error())
	}
	defer r.Close()

	Convey("while fetching or creating account", t, func() {
		Convey("it should have old id", func() {
			a := NewAccount()
			err := a.FetchOrCreate()
			So(err, ShouldNotBeNil)
			So(err, ShouldEqual, ErrOldIdIsNotSet)
		})

		Convey("it should have error if nick contains guest-", func() {
			a := NewAccount()
			a.OldId = "oldestId"
			a.Nick = "guest-test"
			err := a.FetchOrCreate()
			So(err, ShouldNotBeNil)
			So(err, ShouldEqual, ErrGuestsAreNotAllowed)
		})

		Convey("it should not have error if required fields are exist", func() {
			a := NewAccount()
			a.OldId = "oldIdOfAccount"
			a.Nick = "WhatANick"
			err := a.FetchOrCreate()
			So(err, ShouldBeNil)
		})

	})
}

func TestAccountFetchAccountById(t *testing.T) {
	r := runner.New("test")
	if err := r.Init(); err != nil {
		t.Fatalf("couldnt start bongo %s", err.Error())
	}
	defer r.Close()

	Convey("while fetching an account by id", t, func() {
		Convey("it should not have error while fething", func() {
			// create account
			acc := createAccountWithTest()
			So(acc.Create(), ShouldBeNil)

			// fetch the account by id of account
			fa, err := FetchAccountById(acc.Id)
			// error should be nil
			// means that fetching is done successfully
			So(err, ShouldBeNil)
			// account in the db should equal to fetched account
			So(fa.Id, ShouldEqual, acc.Id)
			So(fa.OldId, ShouldEqual, acc.OldId)
			So(fa.Nick, ShouldEqual, acc.Nick)
		})

		Convey("it should have error if record is not found", func() {
			// init account
			a := NewAccount()
			a.Id = 12345
			a.OldId = "oldIdOfAccount"
			a.Nick = "WhatANick"

			// this account id is not exist
			_, err := FetchAccountById(a.Id)
			So(err, ShouldNotBeNil)
			So(err, ShouldEqual, bongo.RecordNotFound)

		})

	})
}

func TestAccountFetchOldIdsByAccountIds(t *testing.T) {
	r := runner.New("test")
	if err := r.Init(); err != nil {
		t.Fatalf("couldnt start bongo %s", err.Error())
	}
	defer r.Close()

	Convey("while fetching Old Ids by account ids", t, func() {
		Convey("it should have account id (ids of length) more than zero", func() {
			acc := []int64{}
			foi, err := FetchOldIdsByAccountIds(acc)
			So(err, ShouldBeNil)
			So(foi, ShouldBeEmpty)
		})

		Convey("it should not have error if account is exist in db", func() {
			// create account
			acc := createAccountWithTest()
			So(acc.Create(), ShouldBeNil)
			// we created slice to send to FetchOldIdsByAccountIds as argument
			idd := []int64{acc.Id}

			// FetchOldIdsByAccountIds returns as slice
			foi, err := FetchOldIdsByAccountIds(idd)
			So(err, ShouldBeNil)
			// used shouldcontain because foi is a slice
			So(foi, ShouldContain, acc.OldId)

		})

		Convey("it should append successfully", func() {
			acc1 := NewAccount()
			acc1.Id = 1
			acc1.OldId = "11"
			acc1.Nick = "acc1"
			So(acc1.Create(), ShouldBeNil)

			acc2 := NewAccount()
			acc2.Id = 2
			acc2.OldId = "22"
			acc2.Nick = "acc2"
			So(acc2.Create(), ShouldBeNil)

			idd := []int64{acc1.Id, acc2.Id}
			old := []string{acc1.OldId, acc2.OldId}

			foi, err := FetchOldIdsByAccountIds(idd)
			So(err, ShouldBeNil)
			So(foi[1], ShouldEqual, old[1])

		})

	})
}

func TestAccountCreateFollowingFeedChannel(t *testing.T) {
	r := runner.New("test")
	if err := r.Init(); err != nil {
		t.Fatalf("couldnt start bongo %s", err.Error())
	}
	defer r.Close()

	Convey("while creating feed channel", t, func() {
		Convey("it should have account id", func() {
			// create account
			acc := NewAccount()

			_, err := acc.CreateFollowingFeedChannel()
			So(err, ShouldNotBeNil)
			So(err, ShouldEqual, ErrAccountIdIsNotSet)
		})

		Convey("it should have creator id as account id", func() {
			// create account
			acc := createAccountWithTest()
			So(acc.Create(), ShouldBeNil)

			cff, err := acc.CreateFollowingFeedChannel()
			So(err, ShouldBeNil)
			So(cff.CreatorId, ShouldEqual, acc.Id)
		})

		Convey("it should have channel name as required", func() {
			// create account
			acc := NewAccount()
			acc.Id = 1
			acc.OldId = "11"
			acc.Nick = "acc1"
			So(acc.Create(), ShouldBeNil)

			cff, err := acc.CreateFollowingFeedChannel()
			So(err, ShouldBeNil)
			So(cff.Name, ShouldEqual, "1-FollowingFeedChannel")
		})

		Convey("it should have group name as Channel_KODING_NAME", func() {
			// create account
			acc := createAccountWithTest()
			So(acc.Create(), ShouldBeNil)

			cff, err := acc.CreateFollowingFeedChannel()
			So(err, ShouldBeNil)
			So(cff.GroupName, ShouldEqual, Channel_KODING_NAME)
		})

		Convey("it should have purpose as Following Feed for me", func() {
			// create account
			acc := createAccountWithTest()
			So(acc.Create(), ShouldBeNil)

			cff, err := acc.CreateFollowingFeedChannel()
			So(err, ShouldBeNil)
			So(cff.Purpose, ShouldEqual, "Following Feed for Me")
		})

		Convey("it should have type constant as Channel_TYPE_FOLLOWERS", func() {
			// create account
			acc := createAccountWithTest()
			So(acc.Create(), ShouldBeNil)

			cff, err := acc.CreateFollowingFeedChannel()
			So(err, ShouldBeNil)
			So(cff.TypeConstant, ShouldEqual, Channel_TYPE_FOLLOWERS)
		})

	})
}

func TestAccountUnMarkAsTroll(t *testing.T) {
	r := runner.New("test")
	if err := r.Init(); err != nil {
		t.Fatalf("couldnt start bongo %s", err.Error())
	}
	defer r.Close()

	Convey("while unmarking as troll", t, func() {
		Convey("it should have account id", func() {
			acc := NewAccount()

			err := acc.UnMarkAsTroll()
			So(err, ShouldNotBeNil)
			So(err, ShouldEqual, ErrAccountIdIsNotSet)
		})

		Convey("it should have account in db", func() {
			acc := NewAccount()
			acc.Id = 1122

			err := acc.UnMarkAsTroll()
			So(err, ShouldNotBeNil)
			So(err, ShouldEqual, bongo.RecordNotFound)
		})
	})
}
<|endoftext|>


<commit_before>package models

import (
	"errors"
	"time"

	"github.com/koding/bongo"
)

type MessageReply struct {
	// unique identifier of the MessageReply
	Id int64 `json:"id"`

	// Id of the interacted message
	MessageId int64 `json:"messageId"`

	// Id of the reply
	ReplyId int64 `json:"replyId"`

	// Creation of the MessageReply
	CreatedAt time.Time `json:"createdAt"`
}

func (m *MessageReply) GetId() int64 {
	return m.Id
}

func (m *MessageReply) TableName() string {
	return "message_reply"
}

func NewMessageReply() *MessageReply {
	return &MessageReply{}
}

func (m *MessageReply) AfterCreate() {
	bongo.B.AfterCreate(m)
}

func (m *MessageReply) AfterUpdate() {
	bongo.B.AfterUpdate(m)
}

func (m *MessageReply) AfterDelete() {
	bongo.B.AfterDelete(m)
}

func (m *MessageReply) Fetch() error {
	return bongo.B.Fetch(m)
}

func (m *MessageReply) Create() error {
	return bongo.B.Create(m)
}

func (m *MessageReply) Delete() error {
	if err := bongo.B.DB.
		Where("message_id = ? and reply_id = ?", m.MessageId, m.ReplyId).
		Delete(m).Error; err != nil {
		return err
	}
	return nil
}

func (m *MessageReply) List() ([]ChannelMessage, error) {
	var replies []int64

	if m.MessageId == 0 {
		return nil, errors.New("MessageId is not set")
	}

	if err := bongo.B.DB.Table(m.TableName()).
		Where("message_id = ?", m.MessageId).
		Pluck("reply_id", &replies).
		Error; err != nil {
		return nil, err
	}

	parent := NewChannelMessage()
	channelMessageReplies, err := parent.FetchByIds(replies)
	if err != nil {
		return nil, err
	}

	return channelMessageReplies, nil
}
<commit_msg>Social: add not null constraints<commit_after>package models

import (
	"errors"
	"time"

	"github.com/koding/bongo"
)

type MessageReply struct {
	// unique identifier of the MessageReply
	Id int64 `json:"id"`

	// Id of the interacted message
	MessageId int64 `json:"messageId"             sql:"NOT NULL"`

	// Id of the reply
	ReplyId int64 `json:"replyId"                 sql:"NOT NULL"`

	// Creation of the MessageReply
	CreatedAt time.Time `json:"createdAt"         sql:"NOT NULL"`
}

func (m *MessageReply) GetId() int64 {
	return m.Id
}

func (m *MessageReply) TableName() string {
	return "message_reply"
}

func NewMessageReply() *MessageReply {
	return &MessageReply{}
}

func (m *MessageReply) AfterCreate() {
	bongo.B.AfterCreate(m)
}

func (m *MessageReply) AfterUpdate() {
	bongo.B.AfterUpdate(m)
}

func (m *MessageReply) AfterDelete() {
	bongo.B.AfterDelete(m)
}

func (m *MessageReply) Fetch() error {
	return bongo.B.Fetch(m)
}

func (m *MessageReply) Create() error {
	return bongo.B.Create(m)
}

func (m *MessageReply) Delete() error {
	if err := bongo.B.DB.
		Where("message_id = ? and reply_id = ?", m.MessageId, m.ReplyId).
		Delete(m).Error; err != nil {
		return err
	}
	return nil
}

func (m *MessageReply) List() ([]ChannelMessage, error) {
	var replies []int64

	if m.MessageId == 0 {
		return nil, errors.New("MessageId is not set")
	}

	if err := bongo.B.DB.Table(m.TableName()).
		Where("message_id = ?", m.MessageId).
		Pluck("reply_id", &replies).
		Error; err != nil {
		return nil, err
	}

	parent := NewChannelMessage()
	channelMessageReplies, err := parent.FetchByIds(replies)
	if err != nil {
		return nil, err
	}

	return channelMessageReplies, nil
}
<|endoftext|>