<commit_before>package controller

import (
	"time"

	"github.com/giantswarm/apiextensions/pkg/apis/provider/v1alpha1"
	gsclient "github.com/giantswarm/apiextensions/pkg/clientset/versioned"
	"github.com/giantswarm/microerror"
	"github.com/giantswarm/micrologger"
	"github.com/giantswarm/operatorkit/client/k8scrdclient"
	"github.com/giantswarm/operatorkit/controller"
	"github.com/giantswarm/operatorkit/informer"
	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
	"k8s.io/client-go/kubernetes"

	"github.com/giantswarm/azure-operator/client"
	"github.com/giantswarm/azure-operator/service/controller/setting"
	"github.com/giantswarm/azure-operator/service/controller/v1"
	"github.com/giantswarm/azure-operator/service/controller/v2"
)

type ClusterConfig struct {
	G8sClient        gsclient.Interface
	InstallationName string
	K8sClient        kubernetes.Interface
	K8sExtClient     apiextensionsclient.Interface
	Logger           micrologger.Logger

	Azure           setting.Azure
	AzureConfig     client.AzureConfig
	ProjectName     string
	TemplateVersion string
}

type Cluster struct {
	*controller.Controller
}

func NewCluster(config ClusterConfig) (*Cluster, error) {
	var err error

	if config.G8sClient == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.G8sClient must not be empty", config)
	}
	if config.K8sClient == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.K8sClient must not be empty", config)
	}
	if config.K8sExtClient == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.K8sExtClient must not be empty", config)
	}
	if config.Logger == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.Logger must not be empty", config)
	}

	if err := config.Azure.Validate(); err != nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.Azure.%s", config, err)
	}
	if err := config.AzureConfig.Validate(); err != nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.AzureConfig.%s", config, err)
	}
	if config.ProjectName == "" {
		return nil, microerror.Maskf(invalidConfigError, "%T.ProjectName must not be empty", config)
	}
	if config.TemplateVersion == "" {
		return nil, microerror.Maskf(invalidConfigError, "%T.TemplateVersion must not be empty", config)
	}

	var crdClient *k8scrdclient.CRDClient
	{
		c := k8scrdclient.Config{
			K8sExtClient: config.K8sExtClient,
			Logger:       config.Logger,
		}

		crdClient, err = k8scrdclient.New(c)
		if err != nil {
			return nil, microerror.Mask(err)
		}
	}

	var newInformer *informer.Informer
	{
		c := informer.Config{
			Logger:  config.Logger,
			Watcher: config.G8sClient.ProviderV1alpha1().AzureConfigs(""),

			//ResyncPeriod: informer.DefaultResyncPeriod,
			ResyncPeriod: 1 * time.Minute,
		}

		newInformer, err = informer.New(c)
		if err != nil {
			return nil, microerror.Mask(err)
		}
	}

	var v1ResourceSet *controller.ResourceSet
	{
		c := v1.ResourceSetConfig{
			K8sClient:    config.K8sClient,
			K8sExtClient: config.K8sExtClient,
			Logger:       config.Logger,

			Azure:            config.Azure,
			AzureConfig:      config.AzureConfig,
			InstallationName: config.InstallationName,
			ProjectName:      config.ProjectName,
			TemplateVersion:  config.TemplateVersion,
		}

		v1ResourceSet, err = v1.NewResourceSet(c)
		if err != nil {
			return nil, microerror.Mask(err)
		}
	}

	var v2ResourceSet *controller.ResourceSet
	{
		c := v2.ResourceSetConfig{
			K8sClient:    config.K8sClient,
			K8sExtClient: config.K8sExtClient,
			Logger:       config.Logger,

			Azure:            config.Azure,
			AzureConfig:      config.AzureConfig,
			InstallationName: config.InstallationName,
			ProjectName:      config.ProjectName,
			TemplateVersion:  config.TemplateVersion,
		}

		v2ResourceSet, err = v2.NewResourceSet(c)
		if err != nil {
			return nil, microerror.Mask(err)
		}
	}

	var resourceRouter *controller.ResourceRouter
	{
		c := controller.ResourceRouterConfig{
			Logger: config.Logger,

			ResourceSets: []*controller.ResourceSet{
				v1ResourceSet,
				v2ResourceSet,
			},
		}

		resourceRouter, err = controller.NewResourceRouter(c)
		if err != nil {
			return nil, microerror.Mask(err)
		}
	}

	var operatorkitController *controller.Controller
	{
		c := controller.Config{
			CRD:            v1alpha1.NewAzureConfigCRD(),
			CRDClient:      crdClient,
			Informer:       newInformer,
			Logger:         config.Logger,
			ResourceRouter: resourceRouter,
			RESTClient:     config.G8sClient.ProviderV1alpha1().RESTClient(),

			Name: config.ProjectName,
		}

		operatorkitController, err = controller.New(c)
		if err != nil {
			return nil, microerror.Mask(err)
		}
	}

	c := &Cluster{
		Controller: operatorkitController,
	}

	return c, nil
}
<commit_msg>sets informer settings explicitly to default values (#218)<commit_after>package controller

import (
	"github.com/giantswarm/apiextensions/pkg/apis/provider/v1alpha1"
	gsclient "github.com/giantswarm/apiextensions/pkg/clientset/versioned"
	"github.com/giantswarm/microerror"
	"github.com/giantswarm/micrologger"
	"github.com/giantswarm/operatorkit/client/k8scrdclient"
	"github.com/giantswarm/operatorkit/controller"
	"github.com/giantswarm/operatorkit/informer"
	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
	"k8s.io/client-go/kubernetes"

	"github.com/giantswarm/azure-operator/client"
	"github.com/giantswarm/azure-operator/service/controller/setting"
	"github.com/giantswarm/azure-operator/service/controller/v1"
	"github.com/giantswarm/azure-operator/service/controller/v2"
)

type ClusterConfig struct {
	G8sClient        gsclient.Interface
	InstallationName string
	K8sClient        kubernetes.Interface
	K8sExtClient     apiextensionsclient.Interface
	Logger           micrologger.Logger

	Azure           setting.Azure
	AzureConfig     client.AzureConfig
	ProjectName     string
	TemplateVersion string
}

type Cluster struct {
	*controller.Controller
}

func NewCluster(config ClusterConfig) (*Cluster, error) {
	var err error

	if config.G8sClient == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.G8sClient must not be empty", config)
	}
	if config.K8sClient == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.K8sClient must not be empty", config)
	}
	if config.K8sExtClient == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.K8sExtClient must not be empty", config)
	}
	if config.Logger == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.Logger must not be empty", config)
	}

	if err := config.Azure.Validate(); err != nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.Azure.%s", config, err)
	}
	if err := config.AzureConfig.Validate(); err != nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.AzureConfig.%s", config, err)
	}
	if config.ProjectName == "" {
		return nil, microerror.Maskf(invalidConfigError, "%T.ProjectName must not be empty", config)
	}
	if config.TemplateVersion == "" {
		return nil, microerror.Maskf(invalidConfigError, "%T.TemplateVersion must not be empty", config)
	}

	var crdClient *k8scrdclient.CRDClient
	{
		c := k8scrdclient.Config{
			K8sExtClient: config.K8sExtClient,
			Logger:       config.Logger,
		}

		crdClient, err = k8scrdclient.New(c)
		if err != nil {
			return nil, microerror.Mask(err)
		}
	}

	var newInformer *informer.Informer
	{
		c := informer.Config{
			Logger:  config.Logger,
			Watcher: config.G8sClient.ProviderV1alpha1().AzureConfigs(""),

			RateWait:     informer.DefaultRateWait,
			ResyncPeriod: informer.DefaultResyncPeriod,
		}

		newInformer, err = informer.New(c)
		if err != nil {
			return nil, microerror.Mask(err)
		}
	}

	var v1ResourceSet *controller.ResourceSet
	{
		c := v1.ResourceSetConfig{
			K8sClient:    config.K8sClient,
			K8sExtClient: config.K8sExtClient,
			Logger:       config.Logger,

			Azure:            config.Azure,
			AzureConfig:      config.AzureConfig,
			InstallationName: config.InstallationName,
			ProjectName:      config.ProjectName,
			TemplateVersion:  config.TemplateVersion,
		}

		v1ResourceSet, err = v1.NewResourceSet(c)
		if err != nil {
			return nil, microerror.Mask(err)
		}
	}

	var v2ResourceSet *controller.ResourceSet
	{
		c := v2.ResourceSetConfig{
			K8sClient:    config.K8sClient,
			K8sExtClient: config.K8sExtClient,
			Logger:       config.Logger,

			Azure:            config.Azure,
			AzureConfig:      config.AzureConfig,
			InstallationName: config.InstallationName,
			ProjectName:      config.ProjectName,
			TemplateVersion:  config.TemplateVersion,
		}

		v2ResourceSet, err = v2.NewResourceSet(c)
		if err != nil {
			return nil, microerror.Mask(err)
		}
	}

	var resourceRouter *controller.ResourceRouter
	{
		c := controller.ResourceRouterConfig{
			Logger: config.Logger,

			ResourceSets: []*controller.ResourceSet{
				v1ResourceSet,
				v2ResourceSet,
			},
		}

		resourceRouter, err = controller.NewResourceRouter(c)
		if err != nil {
			return nil, microerror.Mask(err)
		}
	}

	var operatorkitController *controller.Controller
	{
		c := controller.Config{
			CRD:            v1alpha1.NewAzureConfigCRD(),
			CRDClient:      crdClient,
			Informer:       newInformer,
			Logger:         config.Logger,
			ResourceRouter: resourceRouter,
			RESTClient:     config.G8sClient.ProviderV1alpha1().RESTClient(),

			Name: config.ProjectName,
		}

		operatorkitController, err = controller.New(c)
		if err != nil {
			return nil, microerror.Mask(err)
		}
	}

	c := &Cluster{
		Controller: operatorkitController,
	}

	return c, nil
}
<|endoftext|>


<commit_before>// +build linux freebsd openbsd darwin solaris

package host

import (
	"golang.org/x/sys/unix"
)

func kernelArch() (string, error) {
	var utsname unix.Utsname
	err := unix.Uname(&utsname)
	return string(utsname.Machine[:]), err
}
<commit_msg>Trim null bytes from kernel arch<commit_after>// +build linux freebsd openbsd darwin solaris

package host

import (
	"bytes"

	"golang.org/x/sys/unix"
)

func kernelArch() (string, error) {
	var utsname unix.Utsname
	err := unix.Uname(&utsname)
	return string(utsname.Machine[:bytes.IndexByte(utsname.Machine[:], 0)]), err
}
<|endoftext|>


<commit_before>package filer

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/util"
	"io"
	"sync"
	"time"

	"github.com/golang/protobuf/proto"
	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util/log_buffer"
)

type MetaAggregator struct {
	filers         []string
	grpcDialOption grpc.DialOption
	MetaLogBuffer  *log_buffer.LogBuffer
	// notifying clients
	ListenersLock sync.Mutex
	ListenersCond *sync.Cond
}

// MetaAggregator only aggregates data "on the fly". The logs are not re-persisted to disk.
// The old data comes from what each LocalMetadata persisted on disk.
func NewMetaAggregator(filers []string, grpcDialOption grpc.DialOption) *MetaAggregator {
	t := &MetaAggregator{
		filers:         filers,
		grpcDialOption: grpcDialOption,
	}
	t.ListenersCond = sync.NewCond(&t.ListenersLock)
	t.MetaLogBuffer = log_buffer.NewLogBuffer(LogFlushInterval, nil, func() {
		t.ListenersCond.Broadcast()
	})
	return t
}

func (ma *MetaAggregator) StartLoopSubscribe(f *Filer, self string) {
	for _, filer := range ma.filers {
		go ma.subscribeToOneFiler(f, self, filer)
	}
}

func (ma *MetaAggregator) subscribeToOneFiler(f *Filer, self string, peer string) {

	/*
		Each filer reads the "filer.store.id", which is the store's signature when filer starts.

		When reading from other filers' local meta changes:
		* if the received change does not contain signature from self, apply the change to current filer store.

		Upon connecting to other filers, need to remember their signature and their offsets.

	*/

	var maybeReplicateMetadataChange func(*filer_pb.SubscribeMetadataResponse)
	lastPersistTime := time.Now()
	lastTsNs := time.Now().Add(-LogFlushInterval).UnixNano()

	peerSignature, err := ma.readFilerStoreSignature(peer)
	for err != nil {
		glog.V(0).Infof("connecting to peer filer %s: %v", peer, err)
		time.Sleep(1357 * time.Millisecond)
		peerSignature, err = ma.readFilerStoreSignature(peer)
	}

	// when filer store is not shared by multiple filers
	if peerSignature != f.Signature {
		if prevTsNs, err := ma.readOffset(f, peer, peerSignature); err == nil {
			lastTsNs = prevTsNs
		}

		glog.V(0).Infof("follow peer: %v, last %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs)
		var counter int64
		var synced bool
		maybeReplicateMetadataChange = func(event *filer_pb.SubscribeMetadataResponse) {
			if err := Replay(f.Store, event); err != nil {
				glog.Errorf("failed to reply metadata change from %v: %v", peer, err)
				return
			}
			counter++
			if lastPersistTime.Add(time.Minute).Before(time.Now()) {
				if err := ma.updateOffset(f, peer, peerSignature, event.TsNs); err == nil {
					if event.TsNs < time.Now().Add(-2*time.Minute).UnixNano() {
						glog.V(0).Infof("sync with %s progressed to: %v %0.2f/sec", peer, time.Unix(0, event.TsNs), float64(counter)/60.0)
					} else if !synced {
						synced = true
						glog.V(0).Infof("synced with %s", peer)
					}
					lastPersistTime = time.Now()
					counter = 0
				} else {
					glog.V(0).Infof("failed to update offset for %v: %v", peer, err)
				}
			}
		}
	}

	processEventFn := func(event *filer_pb.SubscribeMetadataResponse) error {
		data, err := proto.Marshal(event)
		if err != nil {
			glog.Errorf("failed to marshal subscribed filer_pb.SubscribeMetadataResponse %+v: %v", event, err)
			return err
		}
		dir := event.Directory
		// println("received meta change", dir, "size", len(data))
		ma.MetaLogBuffer.AddToBuffer([]byte(dir), data, 0)
		if maybeReplicateMetadataChange != nil {
			maybeReplicateMetadataChange(event)
		}
		return nil
	}

	for {
		err := pb.WithFilerClient(peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
			ctx, cancel := context.WithCancel(context.Background())
			defer cancel()
			stream, err := client.SubscribeLocalMetadata(ctx, &filer_pb.SubscribeMetadataRequest{
				ClientName: "filer:" + self,
				PathPrefix: "/",
				SinceNs:    lastTsNs,
			})
			if err != nil {
				return fmt.Errorf("subscribe: %v", err)
			}

			for {
				resp, listenErr := stream.Recv()
				if listenErr == io.EOF {
					return nil
				}
				if listenErr != nil {
					return listenErr
				}

				if err := processEventFn(resp); err != nil {
					return fmt.Errorf("process %v: %v", resp, err)
				}
				lastTsNs = resp.TsNs

				f.onMetadataChangeEvent(resp)

			}
		})
		if err != nil {
			glog.V(0).Infof("subscribing remote %s meta change: %v", peer, err)
			time.Sleep(1733 * time.Millisecond)
		}
	}
}

func (ma *MetaAggregator) readFilerStoreSignature(peer string) (sig int32, err error) {
	err = pb.WithFilerClient(peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
		resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
		if err != nil {
			return err
		}
		sig = resp.Signature
		return nil
	})
	return
}

const (
	MetaOffsetPrefix = "Meta"
)

func (ma *MetaAggregator) readOffset(f *Filer, peer string, peerSignature int32) (lastTsNs int64, err error) {

	key := []byte(MetaOffsetPrefix + "xxxx")
	util.Uint32toBytes(key[len(MetaOffsetPrefix):], uint32(peerSignature))

	value, err := f.Store.KvGet(context.Background(), key)

	if err == ErrKvNotFound {
		glog.Warningf("readOffset %s not found", peer)
		return 0, nil
	}

	if err != nil {
		return 0, fmt.Errorf("readOffset %s : %v", peer, err)
	}

	lastTsNs = int64(util.BytesToUint64(value))

	glog.V(0).Infof("readOffset %s : %d", peer, lastTsNs)

	return
}

func (ma *MetaAggregator) updateOffset(f *Filer, peer string, peerSignature int32, lastTsNs int64) (err error) {

	key := []byte(MetaOffsetPrefix + "xxxx")
	util.Uint32toBytes(key[len(MetaOffsetPrefix):], uint32(peerSignature))

	value := make([]byte, 8)
	util.Uint64toBytes(value, uint64(lastTsNs))

	err = f.Store.KvPut(context.Background(), key, value)

	if err != nil {
		return fmt.Errorf("updateOffset %s : %v", peer, err)
	}

	glog.V(4).Infof("updateOffset %s : %d", peer, lastTsNs)

	return
}
<commit_msg>filer: bootstrap from peer filer<commit_after>package filer

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/util"
	"io"
	"sync"
	"time"

	"github.com/golang/protobuf/proto"
	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util/log_buffer"
)

type MetaAggregator struct {
	filers         []string
	grpcDialOption grpc.DialOption
	MetaLogBuffer  *log_buffer.LogBuffer
	// notifying clients
	ListenersLock sync.Mutex
	ListenersCond *sync.Cond
}

// MetaAggregator only aggregates data "on the fly". The logs are not re-persisted to disk.
// The old data comes from what each LocalMetadata persisted on disk.
func NewMetaAggregator(filers []string, grpcDialOption grpc.DialOption) *MetaAggregator {
	t := &MetaAggregator{
		filers:         filers,
		grpcDialOption: grpcDialOption,
	}
	t.ListenersCond = sync.NewCond(&t.ListenersLock)
	t.MetaLogBuffer = log_buffer.NewLogBuffer(LogFlushInterval, nil, func() {
		t.ListenersCond.Broadcast()
	})
	return t
}

func (ma *MetaAggregator) StartLoopSubscribe(f *Filer, self string) {
	for _, filer := range ma.filers {
		go ma.subscribeToOneFiler(f, self, filer)
	}
}

func (ma *MetaAggregator) subscribeToOneFiler(f *Filer, self string, peer string) {

	/*
		Each filer reads the "filer.store.id", which is the store's signature when filer starts.

		When reading from other filers' local meta changes:
		* if the received change does not contain signature from self, apply the change to current filer store.

		Upon connecting to other filers, need to remember their signature and their offsets.

	*/

	var maybeReplicateMetadataChange func(*filer_pb.SubscribeMetadataResponse)
	lastPersistTime := time.Now()
	lastTsNs := time.Now().Add(-LogFlushInterval).UnixNano()

	peerSignature, err := ma.readFilerStoreSignature(peer)
	for err != nil {
		glog.V(0).Infof("connecting to peer filer %s: %v", peer, err)
		time.Sleep(1357 * time.Millisecond)
		peerSignature, err = ma.readFilerStoreSignature(peer)
	}

	// when filer store is not shared by multiple filers
	if peerSignature != f.Signature {
		lastTsNs = 0
		if prevTsNs, err := ma.readOffset(f, peer, peerSignature); err == nil {
			lastTsNs = prevTsNs
		}

		glog.V(0).Infof("follow peer: %v, last %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs)
		var counter int64
		var synced bool
		maybeReplicateMetadataChange = func(event *filer_pb.SubscribeMetadataResponse) {
			if err := Replay(f.Store, event); err != nil {
				glog.Errorf("failed to reply metadata change from %v: %v", peer, err)
				return
			}
			counter++
			if lastPersistTime.Add(time.Minute).Before(time.Now()) {
				if err := ma.updateOffset(f, peer, peerSignature, event.TsNs); err == nil {
					if event.TsNs < time.Now().Add(-2*time.Minute).UnixNano() {
						glog.V(0).Infof("sync with %s progressed to: %v %0.2f/sec", peer, time.Unix(0, event.TsNs), float64(counter)/60.0)
					} else if !synced {
						synced = true
						glog.V(0).Infof("synced with %s", peer)
					}
					lastPersistTime = time.Now()
					counter = 0
				} else {
					glog.V(0).Infof("failed to update offset for %v: %v", peer, err)
				}
			}
		}
	}

	processEventFn := func(event *filer_pb.SubscribeMetadataResponse) error {
		data, err := proto.Marshal(event)
		if err != nil {
			glog.Errorf("failed to marshal subscribed filer_pb.SubscribeMetadataResponse %+v: %v", event, err)
			return err
		}
		dir := event.Directory
		// println("received meta change", dir, "size", len(data))
		ma.MetaLogBuffer.AddToBuffer([]byte(dir), data, 0)
		if maybeReplicateMetadataChange != nil {
			maybeReplicateMetadataChange(event)
		}
		return nil
	}

	for {
		err := pb.WithFilerClient(peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
			ctx, cancel := context.WithCancel(context.Background())
			defer cancel()
			stream, err := client.SubscribeLocalMetadata(ctx, &filer_pb.SubscribeMetadataRequest{
				ClientName: "filer:" + self,
				PathPrefix: "/",
				SinceNs:    lastTsNs,
			})
			if err != nil {
				return fmt.Errorf("subscribe: %v", err)
			}

			for {
				resp, listenErr := stream.Recv()
				if listenErr == io.EOF {
					return nil
				}
				if listenErr != nil {
					return listenErr
				}

				if err := processEventFn(resp); err != nil {
					return fmt.Errorf("process %v: %v", resp, err)
				}
				lastTsNs = resp.TsNs

				f.onMetadataChangeEvent(resp)

			}
		})
		if err != nil {
			glog.V(0).Infof("subscribing remote %s meta change: %v", peer, err)
			time.Sleep(1733 * time.Millisecond)
		}
	}
}

func (ma *MetaAggregator) readFilerStoreSignature(peer string) (sig int32, err error) {
	err = pb.WithFilerClient(peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
		resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
		if err != nil {
			return err
		}
		sig = resp.Signature
		return nil
	})
	return
}

const (
	MetaOffsetPrefix = "Meta"
)

func (ma *MetaAggregator) readOffset(f *Filer, peer string, peerSignature int32) (lastTsNs int64, err error) {

	key := []byte(MetaOffsetPrefix + "xxxx")
	util.Uint32toBytes(key[len(MetaOffsetPrefix):], uint32(peerSignature))

	value, err := f.Store.KvGet(context.Background(), key)

	if err == ErrKvNotFound {
		glog.Warningf("readOffset %s not found", peer)
		return 0, nil
	}

	if err != nil {
		return 0, fmt.Errorf("readOffset %s : %v", peer, err)
	}

	lastTsNs = int64(util.BytesToUint64(value))

	glog.V(0).Infof("readOffset %s : %d", peer, lastTsNs)

	return
}

func (ma *MetaAggregator) updateOffset(f *Filer, peer string, peerSignature int32, lastTsNs int64) (err error) {

	key := []byte(MetaOffsetPrefix + "xxxx")
	util.Uint32toBytes(key[len(MetaOffsetPrefix):], uint32(peerSignature))

	value := make([]byte, 8)
	util.Uint64toBytes(value, uint64(lastTsNs))

	err = f.Store.KvPut(context.Background(), key, value)

	if err != nil {
		return fmt.Errorf("updateOffset %s : %v", peer, err)
	}

	glog.V(4).Infof("updateOffset %s : %d", peer, lastTsNs)

	return
}
<|endoftext|>


<commit_before>package mount

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/hanwen/go-fuse/v2/fuse"
	"os"
	"time"
)

func (wfs *WFS) Mkdir(cancel <-chan struct{}, in *fuse.MkdirIn, name string, out *fuse.EntryOut) (code fuse.Status) {

	if s := checkName(name); s != fuse.OK {
		return s
	}

	newEntry := &filer_pb.Entry{
		Name:        name,
		IsDirectory: true,
		Attributes: &filer_pb.FuseAttributes{
			Mtime:    time.Now().Unix(),
			Crtime:   time.Now().Unix(),
			FileMode: uint32(os.ModeDir) | in.Mode&^uint32(wfs.option.Umask),
			Uid:      in.Uid,
			Gid:      in.Gid,
		},
	}

	dirFullPath := wfs.inodeToPath.GetPath(in.NodeId)

	entryFullPath := dirFullPath.Child(name)

	err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {

		wfs.mapPbIdFromLocalToFiler(newEntry)
		defer wfs.mapPbIdFromFilerToLocal(newEntry)

		request := &filer_pb.CreateEntryRequest{
			Directory:  string(dirFullPath),
			Entry:      newEntry,
			Signatures: []int32{wfs.signature},
		}

		glog.V(1).Infof("mkdir: %v", request)
		if err := filer_pb.CreateEntry(client, request); err != nil {
			glog.V(0).Infof("mkdir %s: %v", entryFullPath, err)
			return err
		}

		if err := wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)); err != nil {
			return fmt.Errorf("local mkdir dir %s: %v", entryFullPath, err)
		}

		return nil
	})

	glog.V(0).Infof("mkdir %s: %v", entryFullPath, err)

	if err != nil {
		return fuse.EIO
	}

	inode := wfs.inodeToPath.GetInode(entryFullPath)

	wfs.outputPbEntry(out, inode, newEntry)

	return fuse.OK

}
<commit_msg>add rmdir<commit_after>package mount

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/hanwen/go-fuse/v2/fuse"
	"os"
	"strings"
	"syscall"
	"time"
)

func (wfs *WFS) Mkdir(cancel <-chan struct{}, in *fuse.MkdirIn, name string, out *fuse.EntryOut) (code fuse.Status) {

	if s := checkName(name); s != fuse.OK {
		return s
	}

	newEntry := &filer_pb.Entry{
		Name:        name,
		IsDirectory: true,
		Attributes: &filer_pb.FuseAttributes{
			Mtime:    time.Now().Unix(),
			Crtime:   time.Now().Unix(),
			FileMode: uint32(os.ModeDir) | in.Mode&^uint32(wfs.option.Umask),
			Uid:      in.Uid,
			Gid:      in.Gid,
		},
	}

	dirFullPath := wfs.inodeToPath.GetPath(in.NodeId)

	entryFullPath := dirFullPath.Child(name)

	err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {

		wfs.mapPbIdFromLocalToFiler(newEntry)
		defer wfs.mapPbIdFromFilerToLocal(newEntry)

		request := &filer_pb.CreateEntryRequest{
			Directory:  string(dirFullPath),
			Entry:      newEntry,
			Signatures: []int32{wfs.signature},
		}

		glog.V(1).Infof("mkdir: %v", request)
		if err := filer_pb.CreateEntry(client, request); err != nil {
			glog.V(0).Infof("mkdir %s: %v", entryFullPath, err)
			return err
		}

		if err := wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)); err != nil {
			return fmt.Errorf("local mkdir dir %s: %v", entryFullPath, err)
		}

		return nil
	})

	glog.V(0).Infof("mkdir %s: %v", entryFullPath, err)

	if err != nil {
		return fuse.EIO
	}

	inode := wfs.inodeToPath.GetInode(entryFullPath)

	wfs.outputPbEntry(out, inode, newEntry)

	return fuse.OK

}

func (wfs *WFS) Rmdir(cancel <-chan struct{}, header *fuse.InHeader, name string) (code fuse.Status) {

	if name == "." {
		return fuse.Status(syscall.EINVAL)
	}
	if name == ".." {
		return fuse.Status(syscall.ENOTEMPTY)
	}

	dirFullPath := wfs.inodeToPath.GetPath(header.NodeId)
	entryFullPath := dirFullPath.Child(name)

	glog.V(3).Infof("remove directory: %v", entryFullPath)
	ignoreRecursiveErr := true // ignore recursion error since the OS should manage it
	err := filer_pb.Remove(wfs, string(dirFullPath), name, true, true, ignoreRecursiveErr, false, []int32{wfs.signature})
	if err != nil {
		glog.V(0).Infof("remove %s: %v", entryFullPath, err)
		if strings.Contains(err.Error(), filer.MsgFailDelNonEmptyFolder) {
			return fuse.Status(syscall.ENOTEMPTY)
		}
		return fuse.ENOENT
	}

	wfs.metaCache.DeleteEntry(context.Background(), entryFullPath)

	return fuse.OK

}
<|endoftext|>


<commit_before>package main

import (
	"log"

	"html/template"
	"net/http"
	"strconv"
	"strings"

	"github.com/elazarl/go-bindata-assetfs"
	"github.com/gin-gonic/contrib/renders/multitemplate"
	"github.com/gin-gonic/contrib/static"
	"github.com/gin-gonic/gin"
	"github.com/nu7hatch/gouuid"
	"github.com/wadahiro/gitss/server/config"
	"github.com/wadahiro/gitss/server/controller"
	"github.com/wadahiro/gitss/server/indexer"
)

func initRouter(config *config.Config, indexer indexer.Indexer) {
	if !config.Debug {
		gin.SetMode(gin.ReleaseMode)
	}

	r := gin.Default()

	r.Use(gin.Recovery())

	r.Use(func(c *gin.Context) {
		c.Set("indexer", indexer)
		c.Set("config", config)
	})

	r.Use(func(c *gin.Context) {
		c.Set("gitDataDir", config.GitDataDir)
	})

	// r.LoadHTMLGlob("server/templates/*")
	r.HTMLRender = loadTemplates("react.html")

	// We can't use router.Static method to use '/' for static files.
	// see https://github.com/gin-gonic/gin/issues/75
	// r.StaticFS("/", assetFS())

	r.Use(func(c *gin.Context) {
		id, _ := uuid.NewV4()
		c.Set("uuid", id)
	})

	apiPrefix := "/api/v1/"

	// add API routes
	r.GET(apiPrefix+"version", func(c *gin.Context) {
		c.JSON(200, map[string]string{
			"version":    Version,
			"commitHash": CommitHash,
		})
	})

	r.GET(apiPrefix+"search", controller.SearchIndex)
	r.GET(apiPrefix+"statistics", controller.GetIndexStatistics)
	r.GET(apiPrefix+"filters", controller.GetBaseFilters)
	r.GET(apiPrefix+"filters/:organization", controller.GetBaseFilters)
	r.GET(apiPrefix+"filters/:organization/:project", controller.GetBaseFilters)
	r.GET(apiPrefix+"filters/:organization/:project/:repository", controller.GetBaseFilters)

	// react server-side rendering
	// react := NewReact(
	// 	"assets/js/bundle.js",
	// 	debugMode,
	// 	r,
	// )
	// r.GET("/", react.Handle)
	// r.GET("/issues", react.Handle)

	r.Use(static.Serve("/", BinaryFileSystem("assets")))

	r.Run(":" + strconv.Itoa(config.Port))
}

func loadTemplates(list ...string) multitemplate.Render {
	r := multitemplate.New()

	for _, x := range list {
		templateString, err := Asset("server/templates/" + x)
		if err != nil {
			log.Fatal(err)
		}

		tmplMessage, err := template.New(x).Parse(string(templateString))
		if err != nil {
			log.Fatal(err)
		}

		r.Add(x, tmplMessage)
	}

	return r
}

type binaryFileSystem struct {
	fs http.FileSystem
}

func (b *binaryFileSystem) Open(name string) (http.File, error) {
	return b.fs.Open(name)
}

func (b *binaryFileSystem) Exists(prefix string, filepath string) bool {
	if p := strings.TrimPrefix(filepath, prefix); len(p) < len(filepath) {
		if _, err := b.fs.Open(p); err != nil {
			return false
		}
		return true
	}
	return false
}

func BinaryFileSystem(root string) *binaryFileSystem {
	fs := &assetfs.AssetFS{Asset, AssetDir, AssetInfo, root}
	return &binaryFileSystem{
		fs,
	}
}
<commit_msg>Fix routing<commit_after>package main

import (
	"log"

	"html/template"
	"net/http"
	"strconv"
	"strings"

	"github.com/elazarl/go-bindata-assetfs"
	"github.com/gin-gonic/contrib/renders/multitemplate"
	"github.com/gin-gonic/contrib/static"
	"github.com/gin-gonic/gin"
	"github.com/nu7hatch/gouuid"
	"github.com/wadahiro/gitss/server/config"
	"github.com/wadahiro/gitss/server/controller"
	"github.com/wadahiro/gitss/server/indexer"
)

func initRouter(config *config.Config, indexer indexer.Indexer) {
	if !config.Debug {
		gin.SetMode(gin.ReleaseMode)
	}

	r := gin.Default()

	r.Use(gin.Recovery())

	r.Use(func(c *gin.Context) {
		c.Set("indexer", indexer)
		c.Set("config", config)
	})

	r.Use(func(c *gin.Context) {
		c.Set("gitDataDir", config.GitDataDir)
	})

	// r.LoadHTMLGlob("server/templates/*")
	r.HTMLRender = loadTemplates("react.html")

	// We can't use router.Static method to use '/' for static files.
	// see https://github.com/gin-gonic/gin/issues/75
	// r.StaticFS("/", assetFS())

	r.Use(func(c *gin.Context) {
		id, _ := uuid.NewV4()
		c.Set("uuid", id)
	})

	apiPrefix := "/api/v1/"

	// add API routes
	r.GET(apiPrefix+"version", func(c *gin.Context) {
		c.JSON(200, map[string]string{
			"version":    Version,
			"commitHash": CommitHash,
		})
	})

	r.GET(apiPrefix+"search", controller.SearchIndex)
	r.GET(apiPrefix+"statistics", controller.GetIndexStatistics)
	r.GET(apiPrefix+"filters", controller.GetBaseFilters)
	r.GET(apiPrefix+"filters/:organization", controller.GetBaseFilters)
	r.GET(apiPrefix+"filters/:organization/:project", controller.GetBaseFilters)
	r.GET(apiPrefix+"filters/:organization/:project/:repository", controller.GetBaseFilters)

	// react server-side rendering
	// react := NewReact(
	// 	"assets/js/bundle.js",
	// 	debugMode,
	// 	r,
	// )
	// r.GET("/", react.Handle)
	// r.GET("/issues", react.Handle)

	r.Use(static.Serve("/", BinaryFileSystem("assets")))
	r.Use(static.Serve("/search", BinaryFileSystem("assets")))

	r.Run(":" + strconv.Itoa(config.Port))
}

func loadTemplates(list ...string) multitemplate.Render {
	r := multitemplate.New()

	for _, x := range list {
		templateString, err := Asset("server/templates/" + x)
		if err != nil {
			log.Fatal(err)
		}

		tmplMessage, err := template.New(x).Parse(string(templateString))
		if err != nil {
			log.Fatal(err)
		}

		r.Add(x, tmplMessage)
	}

	return r
}

type binaryFileSystem struct {
	fs http.FileSystem
}

func (b *binaryFileSystem) Open(name string) (http.File, error) {
	return b.fs.Open(name)
}

func (b *binaryFileSystem) Exists(prefix string, filepath string) bool {
	if p := strings.TrimPrefix(filepath, prefix); len(p) < len(filepath) {
		if _, err := b.fs.Open(p); err != nil {
			return false
		}
		return true
	}
	return false
}

func BinaryFileSystem(root string) *binaryFileSystem {
	fs := &assetfs.AssetFS{Asset, AssetDir, AssetInfo, root}
	return &binaryFileSystem{
		fs,
	}
}
<|endoftext|>


<commit_before>package server

import (
	"code.google.com/p/go.net/websocket"
	"fmt"
	"github.com/golang/glog"
	"github.com/nelhage/livegrep/client"
	"time"
)

type searchConnection struct {
	srv        *server
	ws         *websocket.Conn
	backend    string
	client     client.Client
	errors     chan error
	incoming   chan Op
	outgoing   chan Op
	shutdown   bool
	lastQuery  *OpQuery
	dispatched time.Time
}

func (s *searchConnection) recvLoop() {
	var op Op
	for {
		if err := OpCodec.Receive(s.ws, &op); err != nil {
			glog.V(1).Infof("Error in receive: %s\n", err.Error())
			if _, ok := err.(*ProtocolError); ok {
				// TODO: is this a good idea?
				// s.outgoing <- &OpError{err.Error()}
				continue
			} else {
				s.errors <- err
				break
			}
		}
		glog.V(2).Infof("Incoming: %s", asJSON{op})
		s.incoming <- op
		if s.shutdown {
			break
		}
	}
	close(s.incoming)
}

func (s *searchConnection) sendLoop() {
	for op := range s.outgoing {
		glog.V(2).Infof("Outgoing: %s", asJSON{op})
		OpCodec.Send(s.ws, op)
	}
}

func query(q *OpQuery) *client.Query {
	return &client.Query{
		Line: q.Line,
		File: q.File,
		Repo: q.Repo,
	}
}

func (s *searchConnection) handle() {
	s.incoming = make(chan Op, 1)
	s.outgoing = make(chan Op, 1)
	s.errors = make(chan error, 1)

	go s.recvLoop()
	go s.sendLoop()
	defer close(s.outgoing)

	var nextQuery *OpQuery

	var search client.Search
	var results <-chan *client.Result
	var err error

SearchLoop:
	for {
		select {
		case op, ok := <-s.incoming:
			if !ok {
				break
			}
			switch t := op.(type) {
			case *OpQuery:
				nextQuery = t
			default:
				s.outgoing <- &OpError{fmt.Sprintf("Invalid opcode %s", op.Opcode())}
				break
			}

		case e := <-s.errors:
			glog.Infof("error reading from client: %s\n", e.Error())
			break SearchLoop
		case res, ok := <-results:
			if ok {
				s.outgoing <- &OpResult{s.lastQuery.Id, res}
			} else {
				st, err := search.Close()
				if err == nil {
					duration := time.Since(s.dispatched)
					glog.Infof("search done remote=%s id=%d query=%s millis=%d",
						s.ws.Request().RemoteAddr,
						s.lastQuery.Id,
						asJSON{query(s.lastQuery)},
						int64(duration/time.Millisecond))
					s.outgoing <- &OpSearchDone{s.lastQuery.Id, int64(duration / time.Millisecond), st}
				} else {
					s.outgoing <- &OpQueryError{s.lastQuery.Id, err.Error()}
				}
				results = nil
				search = nil
			}
		}
		if nextQuery != nil && results == nil {
			if !s.shouldDispatch(nextQuery) {
				nextQuery = nil
				continue
			}
			if err := s.connectBackend(nextQuery.Backend); err != nil {
				s.outgoing <- &OpQueryError{nextQuery.Id, err.Error()}
				nextQuery = nil
				continue
			}
			q := query(nextQuery)
			glog.Infof("dispatching remote=%s id=%d query=%s",
				s.ws.Request().RemoteAddr,
				nextQuery.Id,
				asJSON{q})
			search, err = s.client.Query(q)
			s.dispatched = time.Now()
			if err != nil {
				s.outgoing <- &OpQueryError{nextQuery.Id, err.Error()}
			} else {
				if search == nil {
					panic("nil search and nil error?")
				}
				s.lastQuery = nextQuery
				results = search.Results()
			}
			nextQuery = nil
		}
	}

	s.shutdown = true
}

func (s *searchConnection) shouldDispatch(q *OpQuery) bool {
	if s.lastQuery == nil {
		return true
	}
	if s.lastQuery.Backend != q.Backend ||
		s.lastQuery.Line != q.Line ||
		s.lastQuery.File != q.File ||
		s.lastQuery.Repo != q.Repo {
		return true
	}
	return false
}

func (s *searchConnection) connectBackend(backend string) error {
	if s.client == nil || s.backend != backend {
		if s.client != nil {
			s.client.Close()
		}
		s.backend = backend
		addr := ""
		for _, bk := range s.srv.config.Backends {
			if bk.Id == backend {
				addr = bk.Addr
				break
			}
		}
		if addr == "" {
			return fmt.Errorf("No such backend: %s", backend)
		}
		s.client = client.ClientWithRetry(func() (client.Client, error) {
			return client.Dial("tcp", addr)
		})
	}
	return nil
}

func (s *server) HandleWebsocket(ws *websocket.Conn) {
	c := &searchConnection{
		srv: s,
		ws:  ws,
	}
	c.handle()
}
<commit_msg>Some refactor of the search server.<commit_after>package server

import (
	"code.google.com/p/go.net/websocket"
	"fmt"
	"github.com/golang/glog"
	"github.com/nelhage/livegrep/client"
	"time"
)

type searchConnection struct {
	srv      *server
	ws       *websocket.Conn
	backend  string
	client   client.Client
	errors   chan error
	incoming chan Op
	outgoing chan Op
	shutdown bool
	q        struct {
		last *OpQuery
		// The time we dispatched 'last'
		t    time.Time
		next *OpQuery
	}
}

func (s *searchConnection) recvLoop() {
	var op Op
	for {
		if err := OpCodec.Receive(s.ws, &op); err != nil {
			glog.V(1).Infof("Error in receive: %s\n", err.Error())
			if _, ok := err.(*ProtocolError); ok {
				// TODO: is this a good idea?
				// s.outgoing <- &OpError{err.Error()}
				continue
			} else {
				s.errors <- err
				break
			}
		}
		glog.V(2).Infof("Incoming: %s", asJSON{op})
		s.incoming <- op
		if s.shutdown {
			break
		}
	}
	close(s.incoming)
}

func (s *searchConnection) sendLoop() {
	for op := range s.outgoing {
		glog.V(2).Infof("Outgoing: %s", asJSON{op})
		OpCodec.Send(s.ws, op)
	}
}

func query(q *OpQuery) *client.Query {
	return &client.Query{
		Line: q.Line,
		File: q.File,
		Repo: q.Repo,
	}
}

func (s *searchConnection) handle() {
	s.incoming = make(chan Op, 1)
	s.outgoing = make(chan Op, 1)
	s.errors = make(chan error, 1)

	go s.recvLoop()
	go s.sendLoop()
	defer close(s.outgoing)

	var search client.Search
	var results <-chan *client.Result
	var err error

SearchLoop:
	for {
		select {
		case op, ok := <-s.incoming:
			if !ok {
				break
			}
			switch t := op.(type) {
			case *OpQuery:
				s.q.next = t
			default:
				s.outgoing <- &OpError{fmt.Sprintf("Invalid opcode %s", op.Opcode())}
				break
			}

		case e := <-s.errors:
			glog.Infof("error reading from client: %s\n", e.Error())
			break SearchLoop
		case res, ok := <-results:
			if ok {
				s.outgoing <- &OpResult{s.q.last.Id, res}
			} else {
				st, err := search.Close()
				if err == nil {
					duration := time.Since(s.q.t)
					glog.Infof("search done remote=%s id=%d query=%s millis=%d",
						s.ws.Request().RemoteAddr,
						s.q.last.Id,
						asJSON{query(s.q.last)},
						int64(duration/time.Millisecond))
					s.outgoing <- &OpSearchDone{s.q.last.Id, int64(duration / time.Millisecond), st}
				} else {
					s.outgoing <- &OpQueryError{s.q.last.Id, err.Error()}
				}
				results = nil
				search = nil
			}
		}
		if s.q.next != nil && results == nil {
			if !s.shouldDispatch(s.q.next) {
				s.q.next = nil
				continue
			}
			if err := s.connectBackend(s.q.next.Backend); err != nil {
				s.outgoing <- &OpQueryError{s.q.next.Id, err.Error()}
				s.q.next = nil
				continue
			}
			q := query(s.q.next)
			glog.Infof("dispatching remote=%s id=%d query=%s",
				s.ws.Request().RemoteAddr,
				s.q.next.Id,
				asJSON{q})
			search, err = s.client.Query(q)
			s.q.t = time.Now()
			if err != nil {
				s.outgoing <- &OpQueryError{s.q.next.Id, err.Error()}
			} else {
				if search == nil {
					panic("nil search and nil error?")
				}
				s.q.last = s.q.next
				results = search.Results()
			}
			s.q.next = nil
		}
	}

	s.shutdown = true
}

func (s *searchConnection) shouldDispatch(q *OpQuery) bool {
	if s.q.last == nil {
		return true
	}
	if s.q.last.Backend != q.Backend ||
		s.q.last.Line != q.Line ||
		s.q.last.File != q.File ||
		s.q.last.Repo != q.Repo {
		return true
	}
	return false
}

func (s *searchConnection) connectBackend(backend string) error {
	if s.client == nil || s.backend != backend {
		if s.client != nil {
			s.client.Close()
		}
		s.backend = backend
		addr := ""
		for _, bk := range s.srv.config.Backends {
			if bk.Id == backend {
				addr = bk.Addr
				break
			}
		}
		if addr == "" {
			return fmt.Errorf("No such backend: %s", backend)
		}
		s.client = client.ClientWithRetry(func() (client.Client, error) {
			return client.Dial("tcp", addr)
		})
	}
	return nil
}

func (s *server) HandleWebsocket(ws *websocket.Conn) {
	c := &searchConnection{
		srv: s,
		ws:  ws,
	}
	c.handle()
}
<|endoftext|>


<commit_before>// Copyright 2015 Tony Bai.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package cmppserver

import (
	"errors"
	"fmt"
	"log"
	"net"
	"time"

	cmppconn "github.com/bigwhite/gocmpp/conn"
	cmpppacket "github.com/bigwhite/gocmpp/packet"
)

//
var (
	ErrEmptyServerAddr = errors.New("cmpp server listen: empty server addr")
	ErrUnsupportedPkt  = errors.New("cmpp server read packet: receive a unsupported pkt")
)

type Packet struct {
	cmpppacket.Packer
	*cmppconn.Conn
}

type Response struct {
	*Packet
	cmpppacket.Packer
	SeqId uint32
}

type Handler interface {
	ServeCmpp(*Response, *Packet) (bool, error)
}

// The HandlerFunc type is an adapter to allow the use of
// ordinary functions as Cmpp handlers.  If f is a function
// with the appropriate signature, HandlerFunc(f) is a
// Handler object that calls f.
type HandlerFunc func(*Response, *Packet) (bool, error)

// ServeHTTP calls f(r, p).
func (f HandlerFunc) ServeCmpp(r *Response, p *Packet) (bool, error) {
	return f(r, p)
}

type Server struct {
	Addr    string
	Handler Handler // handler to invoke, protocolValidator if nil
	Typ     cmppconn.Type

	// ErrorLog specifies an optional logger for errors accepting
	// connections and unexpected behavior from handlers.
	// If nil, logging goes to os.Stderr via the log package's
	// standard logger.
	ErrorLog *log.Logger
}

// A conn represents the server side of a Cmpp connection.
type conn struct {
	*cmppconn.Conn
	remoteAddr string  // network address of remote side
	server     *Server // the Server on which the connection arrived
}

// Serve accepts incoming connections on the Listener l, creating a
// new service goroutine for each.  The service goroutines read requests and
// then call srv.Handler to reply to them.
func (srv *Server) Serve(l net.Listener) error {
	defer l.Close()
	var tempDelay time.Duration // how long to sleep on accept failure
	for {
		rw, e := l.Accept()
		if e != nil {
			if ne, ok := e.(net.Error); ok && ne.Temporary() {
				if tempDelay == 0 {
					tempDelay = 5 * time.Millisecond
				} else {
					tempDelay *= 2
				}
				if max := 1 * time.Second; tempDelay > max {
					tempDelay = max
				}
				srv.ErrorLog.Printf("cmpp: Accept error: %v; retrying in %v", e, tempDelay)
				time.Sleep(tempDelay)
				continue
			}
			return e
		}
		tempDelay = 0
		c, err := srv.newConn(rw)
		if err != nil {
			continue
		}
		go c.serve()
	}
}

func (c *conn) readPacket() (*Response, error) {
	i, err := c.Conn.RecvAndUnpackPkt()
	if err != nil {
		return nil, err
	}
	typ := c.server.Typ

	var pkt *Packet
	var rsp *Response
	switch p := i.(type) {
	case *cmpppacket.CmppConnReqPkt:
		pkt = &Packet{
			Packer: p,
			Conn:   c.Conn,
		}

		if typ == cmppconn.V30 {
			rsp = &Response{
				Packet: pkt,
				Packer: &cmpppacket.Cmpp3ConnRspPkt{
					SeqId: p.SeqId,
				},
				SeqId: p.SeqId,
			}
		} else {
			rsp = &Response{
				Packet: pkt,
				Packer: &cmpppacket.Cmpp2ConnRspPkt{
					SeqId: p.SeqId,
				},
				SeqId: p.SeqId,
			}
		}

	case *cmpppacket.Cmpp2SubmitReqPkt:
		pkt = &Packet{
			Packer: p,
			Conn:   c.Conn,
		}

		rsp = &Response{
			Packet: pkt,
			Packer: &cmpppacket.Cmpp2SubmitRspPkt{
				SeqId: p.SeqId,
			},
			SeqId: p.SeqId,
		}

	case *cmpppacket.Cmpp3SubmitReqPkt:
		pkt = &Packet{
			Packer: p,
			Conn:   c.Conn,
		}

		rsp = &Response{
			Packet: pkt,
			Packer: &cmpppacket.Cmpp3SubmitRspPkt{
				SeqId: p.SeqId,
			},
			SeqId: p.SeqId,
		}

	case *cmpppacket.Cmpp2FwdReqPkt:
		pkt = &Packet{
			Packer: p,
			Conn:   c.Conn,
		}

		rsp = &Response{
			Packet: pkt,
			Packer: &cmpppacket.Cmpp2FwdRspPkt{
				SeqId: p.SeqId,
			},
			SeqId: p.SeqId,
		}

	case *cmpppacket.Cmpp3FwdReqPkt:
		pkt = &Packet{
			Packer: p,
			Conn:   c.Conn,
		}

		rsp = &Response{
			Packet: pkt,
			Packer: &cmpppacket.Cmpp3FwdRspPkt{
				SeqId: p.SeqId,
			},
			SeqId: p.SeqId,
		}

	case *cmpppacket.Cmpp2DeliverRspPkt:
		pkt = &Packet{
			Packer: p,
			Conn:   c.Conn,
		}

		rsp = &Response{
			Packet: pkt,
		}

	case *cmpppacket.Cmpp3DeliverRspPkt:
		pkt = &Packet{
			Packer: p,
			Conn:   c.Conn,
		}

		rsp = &Response{
			Packet: pkt,
		}

	case *cmpppacket.CmppActiveTestReqPkt:
		pkt = &Packet{
			Packer: p,
			Conn:   c.Conn,
		}

		rsp = &Response{
			Packet: pkt,
			Packer: &cmpppacket.CmppActiveTestRspPkt{
				SeqId: p.SeqId,
			},
			SeqId: p.SeqId,
		}

	case *cmpppacket.CmppActiveTestRspPkt:
		pkt = &Packet{
			Packer: p,
			Conn:   c.Conn,
		}

		rsp = &Response{
			Packet: pkt,
		}

	case *cmpppacket.CmppTerminateReqPkt:
		pkt = &Packet{
			Packer: p,
			Conn:   c.Conn,
		}

		rsp = &Response{
			Packet: pkt,
			Packer: &cmpppacket.CmppTerminateRspPkt{
				SeqId: p.SeqId,
			},
			SeqId: p.SeqId,
		}

	case *cmpppacket.CmppTerminateRspPkt:
		pkt = &Packet{
			Packer: p,
			Conn:   c.Conn,
		}

		rsp = &Response{
			Packet: pkt,
		}
	default:
		return nil, cmpppacket.NewOpError(ErrUnsupportedPkt,
			fmt.Sprintf("readPacket: receive unsupported packet type: %#v", p))
	}
	return rsp, nil
}

// Close the connection.
func (c *conn) close() {
	p := &cmpppacket.CmppTerminateReqPkt{}

	err := c.Conn.SendPkt(p, <-c.Conn.SeqId)
	if err != nil {
		c.server.ErrorLog.Printf("cmpp: close connection error: %v\n", err)
	}
	c.Conn.Close()
}

func (c *conn) finishPacket(r *Response) error {
	if r.Packer == nil {
		return nil
	}

	return c.Conn.SendPkt(r.Packer, r.SeqId)
}

// Serve a new connection.
func (c *conn) serve() {
	defer func() {
		if err := recover(); err != nil {
			c.server.ErrorLog.Printf("cmpp: panic serving %v: %v\n", c.remoteAddr, err)
		}
	}()

	for {
		r, err := c.readPacket()
		if err != nil {
			c.Close()
			break
		}

		_, err = c.server.Handler.ServeCmpp(r, r.Packet)
		if err != nil {
			c.Close()
			break
		}

		if err := c.finishPacket(r); err != nil {
			c.Close()
			break
		}
	}
}

// Create new connection from rwc.
func (srv *Server) newConn(rwc net.Conn) (c *conn, err error) {
	c = new(conn)
	c.remoteAddr = rwc.RemoteAddr().String()
	c.server = srv
	c.Conn = cmppconn.New(rwc, srv.Typ)
	c.Conn.SetState(cmppconn.CONN_CONNECTED)
	return c, nil
}

func (srv *Server) listenAndServe() error {
	if srv.Addr == "" {
		return ErrEmptyServerAddr
	}
	ln, err := net.Listen("tcp", srv.Addr)
	if err != nil {
		return err
	}
	return srv.Serve(tcpKeepAliveListener{ln.(*net.TCPListener)})
}

func protocolValidate(r *Response, p *Packet) (bool, error) {
	return false, nil
}

// ListenAndServe listens on the TCP network address addr
// and then calls Serve with handler to handle requests.
func ListenAndServe(addr string, typ cmppconn.Type, handlers ...Handler) error {
	if addr == "" {
		return ErrEmptyServerAddr
	}

	var handler Handler
	if handlers != nil {
		handler = HandlerFunc(func(r *Response, p *Packet) (bool, error) {
			for _, h := range handlers {
				next, err := h.ServeCmpp(r, p)
				if err != nil || !next {
					return next, err
				}
			}
			return false, nil
		})
	} else {
		handler = HandlerFunc(protocolValidate)
	}
	server := &Server{Addr: addr, Handler: handler, Typ: typ}
	return server.listenAndServe()
}

// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted
// connections. It's used by ListenAndServe so
// dead TCP connections (e.g. closing laptop mid-download) eventually
// go away. the tcpKeepAliveListener's implementation is copied from
// http package.
type tcpKeepAliveListener struct {
	*net.TCPListener
}

func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) {
	tc, err := ln.AcceptTCP()
	if err != nil {
		return
	}
	tc.SetKeepAlive(true)
	tc.SetKeepAlivePeriod(1 * time.Minute) // 1min
	return tc, nil
}
<commit_msg>conn.Close to conn.close<commit_after>// Copyright 2015 Tony Bai.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package cmppserver

import (
	"errors"
	"fmt"
	"log"
	"net"
	"time"

	cmppconn "github.com/bigwhite/gocmpp/conn"
	cmpppacket "github.com/bigwhite/gocmpp/packet"
)

//
var (
	ErrEmptyServerAddr = errors.New("cmpp server listen: empty server addr")
	ErrUnsupportedPkt  = errors.New("cmpp server read packet: receive a unsupported pkt")
)

type Packet struct {
	cmpppacket.Packer
	*cmppconn.Conn
}

type Response struct {
	*Packet
	cmpppacket.Packer
	SeqId uint32
}

type Handler interface {
	ServeCmpp(*Response, *Packet) (bool, error)
}

// The HandlerFunc type is an adapter to allow the use of
// ordinary functions as Cmpp handlers.  If f is a function
// with the appropriate signature, HandlerFunc(f) is a
// Handler object that calls f.
type HandlerFunc func(*Response, *Packet) (bool, error)

// ServeHTTP calls f(r, p).
func (f HandlerFunc) ServeCmpp(r *Response, p *Packet) (bool, error) {
	return f(r, p)
}

type Server struct {
	Addr    string
	Handler Handler // handler to invoke, protocolValidator if nil
	Typ     cmppconn.Type

	// ErrorLog specifies an optional logger for errors accepting
	// connections and unexpected behavior from handlers.
	// If nil, logging goes to os.Stderr via the log package's
	// standard logger.
	ErrorLog *log.Logger
}

// A conn represents the server side of a Cmpp connection.
type conn struct {
	*cmppconn.Conn
	remoteAddr string  // network address of remote side
	server     *Server // the Server on which the connection arrived
}

// Serve accepts incoming connections on the Listener l, creating a
// new service goroutine for each.  The service goroutines read requests and
// then call srv.Handler to reply to them.
func (srv *Server) Serve(l net.Listener) error {
	defer l.Close()
	var tempDelay time.Duration // how long to sleep on accept failure
	for {
		rw, e := l.Accept()
		if e != nil {
			if ne, ok := e.(net.Error); ok && ne.Temporary() {
				if tempDelay == 0 {
					tempDelay = 5 * time.Millisecond
				} else {
					tempDelay *= 2
				}
				if max := 1 * time.Second; tempDelay > max {
					tempDelay = max
				}
				srv.ErrorLog.Printf("cmpp: Accept error: %v; retrying in %v", e, tempDelay)
				time.Sleep(tempDelay)
				continue
			}
			return e
		}
		tempDelay = 0
		c, err := srv.newConn(rw)
		if err != nil {
			continue
		}
		go c.serve()
	}
}

func (c *conn) readPacket() (*Response, error) {
	i, err := c.Conn.RecvAndUnpackPkt()
	if err != nil {
		return nil, err
	}
	typ := c.server.Typ

	var pkt *Packet
	var rsp *Response
	switch p := i.(type) {
	case *cmpppacket.CmppConnReqPkt:
		pkt = &Packet{
			Packer: p,
			Conn:   c.Conn,
		}

		if typ == cmppconn.V30 {
			rsp = &Response{
				Packet: pkt,
				Packer: &cmpppacket.Cmpp3ConnRspPkt{
					SeqId: p.SeqId,
				},
				SeqId: p.SeqId,
			}
		} else {
			rsp = &Response{
				Packet: pkt,
				Packer: &cmpppacket.Cmpp2ConnRspPkt{
					SeqId: p.SeqId,
				},
				SeqId: p.SeqId,
			}
		}

	case *cmpppacket.Cmpp2SubmitReqPkt:
		pkt = &Packet{
			Packer: p,
			Conn:   c.Conn,
		}

		rsp = &Response{
			Packet: pkt,
			Packer: &cmpppacket.Cmpp2SubmitRspPkt{
				SeqId: p.SeqId,
			},
			SeqId: p.SeqId,
		}

	case *cmpppacket.Cmpp3SubmitReqPkt:
		pkt = &Packet{
			Packer: p,
			Conn:   c.Conn,
		}

		rsp = &Response{
			Packet: pkt,
			Packer: &cmpppacket.Cmpp3SubmitRspPkt{
				SeqId: p.SeqId,
			},
			SeqId: p.SeqId,
		}

	case *cmpppacket.Cmpp2FwdReqPkt:
		pkt = &Packet{
			Packer: p,
			Conn:   c.Conn,
		}

		rsp = &Response{
			Packet: pkt,
			Packer: &cmpppacket.Cmpp2FwdRspPkt{
				SeqId: p.SeqId,
			},
			SeqId: p.SeqId,
		}

	case *cmpppacket.Cmpp3FwdReqPkt:
		pkt = &Packet{
			Packer: p,
			Conn:   c.Conn,
		}

		rsp = &Response{
			Packet: pkt,
			Packer: &cmpppacket.Cmpp3FwdRspPkt{
				SeqId: p.SeqId,
			},
			SeqId: p.SeqId,
		}

	case *cmpppacket.Cmpp2DeliverRspPkt:
		pkt = &Packet{
			Packer: p,
			Conn:   c.Conn,
		}

		rsp = &Response{
			Packet: pkt,
		}

	case *cmpppacket.Cmpp3DeliverRspPkt:
		pkt = &Packet{
			Packer: p,
			Conn:   c.Conn,
		}

		rsp = &Response{
			Packet: pkt,
		}

	case *cmpppacket.CmppActiveTestReqPkt:
		pkt = &Packet{
			Packer: p,
			Conn:   c.Conn,
		}

		rsp = &Response{
			Packet: pkt,
			Packer: &cmpppacket.CmppActiveTestRspPkt{
				SeqId: p.SeqId,
			},
			SeqId: p.SeqId,
		}

	case *cmpppacket.CmppActiveTestRspPkt:
		pkt = &Packet{
			Packer: p,
			Conn:   c.Conn,
		}

		rsp = &Response{
			Packet: pkt,
		}

	case *cmpppacket.CmppTerminateReqPkt:
		pkt = &Packet{
			Packer: p,
			Conn:   c.Conn,
		}

		rsp = &Response{
			Packet: pkt,
			Packer: &cmpppacket.CmppTerminateRspPkt{
				SeqId: p.SeqId,
			},
			SeqId: p.SeqId,
		}

	case *cmpppacket.CmppTerminateRspPkt:
		pkt = &Packet{
			Packer: p,
			Conn:   c.Conn,
		}

		rsp = &Response{
			Packet: pkt,
		}
	default:
		return nil, cmpppacket.NewOpError(ErrUnsupportedPkt,
			fmt.Sprintf("readPacket: receive unsupported packet type: %#v", p))
	}
	return rsp, nil
}

// Close the connection.
func (c *conn) close() {
	p := &cmpppacket.CmppTerminateReqPkt{}

	err := c.Conn.SendPkt(p, <-c.Conn.SeqId)
	if err != nil {
		c.server.ErrorLog.Printf("cmpp: close connection error: %v\n", err)
	}
	c.Conn.Close()
}

func (c *conn) finishPacket(r *Response) error {
	if r.Packer == nil {
		return nil
	}

	return c.Conn.SendPkt(r.Packer, r.SeqId)
}

// Serve a new connection.
func (c *conn) serve() {
	defer func() {
		if err := recover(); err != nil {
			c.server.ErrorLog.Printf("cmpp: panic serving %v: %v\n", c.remoteAddr, err)
		}
	}()

	for {
		r, err := c.readPacket()
		if err != nil {
			c.close()
			break
		}

		_, err = c.server.Handler.ServeCmpp(r, r.Packet)
		if err != nil {
			c.close()
			break
		}

		if err := c.finishPacket(r); err != nil {
			c.close()
			break
		}
	}
}

// Create new connection from rwc.
func (srv *Server) newConn(rwc net.Conn) (c *conn, err error) {
	c = new(conn)
	c.remoteAddr = rwc.RemoteAddr().String()
	c.server = srv
	c.Conn = cmppconn.New(rwc, srv.Typ)
	c.Conn.SetState(cmppconn.CONN_CONNECTED)
	return c, nil
}

func (srv *Server) listenAndServe() error {
	if srv.Addr == "" {
		return ErrEmptyServerAddr
	}
	ln, err := net.Listen("tcp", srv.Addr)
	if err != nil {
		return err
	}
	return srv.Serve(tcpKeepAliveListener{ln.(*net.TCPListener)})
}

func protocolValidate(r *Response, p *Packet) (bool, error) {
	return false, nil
}

// ListenAndServe listens on the TCP network address addr
// and then calls Serve with handler to handle requests.
func ListenAndServe(addr string, typ cmppconn.Type, handlers ...Handler) error {
	if addr == "" {
		return ErrEmptyServerAddr
	}

	var handler Handler
	if handlers != nil {
		handler = HandlerFunc(func(r *Response, p *Packet) (bool, error) {
			for _, h := range handlers {
				next, err := h.ServeCmpp(r, p)
				if err != nil || !next {
					return next, err
				}
			}
			return false, nil
		})
	} else {
		handler = HandlerFunc(protocolValidate)
	}
	server := &Server{Addr: addr, Handler: handler, Typ: typ}
	return server.listenAndServe()
}

// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted
// connections. It's used by ListenAndServe so
// dead TCP connections (e.g. closing laptop mid-download) eventually
// go away. the tcpKeepAliveListener's implementation is copied from
// http package.
type tcpKeepAliveListener struct {
	*net.TCPListener
}

func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) {
	tc, err := ln.AcceptTCP()
	if err != nil {
		return
	}
	tc.SetKeepAlive(true)
	tc.SetKeepAlivePeriod(1 * time.Minute) // 1min
	return tc, nil
}
<|endoftext|>


<commit_before>package server

import (
	"log"
	"net"
	"time"

	pb "github.com/joshrotenberg/grpc-cache/cache"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	"github.com/joshrotenberg/grpc-cache/lru"
)

// CacheServer encapsulates the cache server things.
type CacheServer struct {
	cache      *lru.Cache
	grpcServer *grpc.Server
	listener   net.Listener
}

func NewWithListener(listener net.Listener, maxEntries int) *CacheServer {
	grpcServer := grpc.NewServer()
	server := CacheServer{
		cache:      lru.New(maxEntries),
		grpcServer: grpcServer,
		listener:   listener,
	}

	pb.RegisterCacheServer(grpcServer, &server)
	return &server
}

// New returns a new instance of the server. It takes host:port and maxEntries arguments,
// which defines the max number of entries allowed in the cache. Set to 0 for unlimited.
func New(address string, maxEntries int) (*CacheServer, error) {
	listener, err := net.Listen("tcp", address)
	if err != nil {
		return nil, err
	}
	return NewWithListener(listener, maxEntries), nil
}

// Start starts the cache server. It takes a host:port string on which to run the server.
func (s *CacheServer) Start() {
	go func() {
		if err := s.grpcServer.Serve(s.listener); err != nil {
			log.Fatalf("failed to serve: %v", err)
		}
	}()
}

// Stop tries to gracefull stop the server.
func (s *CacheServer) Stop() {
	s.grpcServer.GracefulStop()
}

func cacheError(err error, op pb.CacheRequest_Operation, key string) error {
	switch err {
	case lru.ErrNotFound:
		return status.Errorf(codes.NotFound, "%s error: '%s' not found", op, key)
	case lru.ErrExists:
		return status.Errorf(codes.AlreadyExists, "%s error: '%s' exists", op, key)
	}
	return err
}

func cacheResponse(err error, op pb.CacheRequest_Operation, item *pb.CacheItem) (*pb.CacheResponse, error) {
	if err != nil {
		return nil, cacheError(err, op, item.Key)
	}
	response := &pb.CacheResponse{
		Item: item,
	}
	return response, err
}

// Call calls the cache operation in in.Operation
func (s *CacheServer) Call(ctx context.Context, in *pb.CacheRequest) (*pb.CacheResponse, error) {

	var err error
	s.cache.Lock()
	defer s.cache.Unlock()

	switch in.Operation {
	case pb.CacheRequest_SET:
		s.cache.Set(in.Item.Key, in.Item.Value, time.Duration(in.Item.Ttl)*time.Second)
		return cacheResponse(nil, in.Operation, &pb.CacheItem{Key: in.Item.Key})
	case pb.CacheRequest_CAS:
		err = s.cache.Cas(in.Item.Key, in.Item.Value, time.Duration(in.Item.Ttl)*time.Second, uint64(in.Item.Cas))
		return cacheResponse(err, in.Operation, &pb.CacheItem{Key: in.Item.Key})
	case pb.CacheRequest_GET:
		value, err := s.cache.Get(in.Item.Key)
		return cacheResponse(err, in.Operation, &pb.CacheItem{Key: in.Item.Key, Value: value})
	case pb.CacheRequest_GETS:
		value, cas, err := s.cache.Gets(in.Item.Key)
		return cacheResponse(err, in.Operation, &pb.CacheItem{Key: in.Item.Key, Value: value, Cas: cas})
	case pb.CacheRequest_ADD:
		err = s.cache.Add(in.Item.Key, in.Item.Value, time.Duration(in.Item.Ttl)*time.Second)
		return cacheResponse(err, in.Operation, &pb.CacheItem{Key: in.Item.Key})
	case pb.CacheRequest_REPLACE:
		err = s.cache.Replace(in.Item.Key, in.Item.Value, time.Duration(in.Item.Ttl)*time.Second)
		return cacheResponse(err, in.Operation, &pb.CacheItem{Key: in.Item.Key})
	case pb.CacheRequest_DELETE:
		s.cache.Delete(in.Item.Key)
		return cacheResponse(nil, in.Operation, &pb.CacheItem{Key: in.Item.Key})
	case pb.CacheRequest_TOUCH:
		err = s.cache.Touch(in.Item.Key, time.Duration(in.Item.Ttl)*time.Second)
		return cacheResponse(err, in.Operation, &pb.CacheItem{Key: in.Item.Key})
	case pb.CacheRequest_APPEND:
		err = s.cache.Append(in.Item.Key, in.Append, time.Duration(in.Item.Ttl)*time.Second)
		return cacheResponse(err, in.Operation, &pb.CacheItem{Key: in.Item.Key})
	case pb.CacheRequest_PREPEND:
		err = s.cache.Prepend(in.Item.Key, in.Prepend, time.Duration(in.Item.Ttl)*time.Second)
		return cacheResponse(err, in.Operation, &pb.CacheItem{Key: in.Item.Key})
	case pb.CacheRequest_INCREMENT:
		err = s.cache.Increment(in.Item.Key, in.Increment)
		return cacheResponse(err, in.Operation, &pb.CacheItem{Key: in.Item.Key})
	case pb.CacheRequest_DECREMENT:
		err = s.cache.Decrement(in.Item.Key, in.Decrement)
		return cacheResponse(err, in.Operation, &pb.CacheItem{Key: in.Item.Key})
	case pb.CacheRequest_FLUSHALL:
		s.cache.FlushAll()
		return cacheResponse(nil, in.Operation, nil)
	default:
		return nil, status.Errorf(codes.Unimplemented, "unrecognized cache command %d", in.Operation)
	}

}

// Set stores a key/value pair in the cache.
func (s *CacheServer) Set(ctx context.Context, in *pb.CacheRequest) (*pb.CacheResponse, error) {
	in.Operation = pb.CacheRequest_SET
	return s.Call(ctx, in)
}

// Cas does a "check and set" or "compare and swap" operation. If the given key exists
// and the currently stored CAS value matches the one supplied, the value is stored.
func (s *CacheServer) Cas(ctx context.Context, in *pb.CacheRequest) (*pb.CacheResponse, error) {
	in.Operation = pb.CacheRequest_CAS
	return s.Call(ctx, in)
}

// Get gets a key/value pair from the cache.
func (s *CacheServer) Get(ctx context.Context, in *pb.CacheRequest) (*pb.CacheResponse, error) {
	in.Operation = pb.CacheRequest_GET
	return s.Call(ctx, in)
}

// Gets gets a key/value pair from the cache and includes its CAS value.
func (s *CacheServer) Gets(ctx context.Context, in *pb.CacheRequest) (*pb.CacheResponse, error) {
	in.Operation = pb.CacheRequest_GETS
	return s.Call(ctx, in)
}

// Add adds a key/value pair to the cache.
func (s *CacheServer) Add(ctx context.Context, in *pb.CacheRequest) (*pb.CacheResponse, error) {
	in.Operation = pb.CacheRequest_ADD
	return s.Call(ctx, in)
}

// Replace replaces a key/value pair in the cache.
func (s *CacheServer) Replace(ctx context.Context, in *pb.CacheRequest) (*pb.CacheResponse, error) {
	in.Operation = pb.CacheRequest_REPLACE
	return s.Call(ctx, in)
}

// Delete deletes a key/value pair from the cache.
func (s *CacheServer) Delete(ctx context.Context, in *pb.CacheRequest) (*pb.CacheResponse, error) {
	in.Operation = pb.CacheRequest_DELETE
	return s.Call(ctx, in)
}

// Touch updates the key/value pair's cas and ttl.
func (s *CacheServer) Touch(ctx context.Context, in *pb.CacheRequest) (*pb.CacheResponse, error) {
	in.Operation = pb.CacheRequest_TOUCH
	return s.Call(ctx, in)
}

// Append appends the value to the current value for the key.
func (s *CacheServer) Append(ctx context.Context, in *pb.CacheRequest) (*pb.CacheResponse, error) {
	in.Operation = pb.CacheRequest_APPEND
	return s.Call(ctx, in)
}

// Prepend prepends the value to the current value for the key.
func (s *CacheServer) Prepend(ctx context.Context, in *pb.CacheRequest) (*pb.CacheResponse, error) {
	in.Operation = pb.CacheRequest_PREPEND
	return s.Call(ctx, in)
}

// Increment increments the value for key.
func (s *CacheServer) Increment(ctx context.Context, in *pb.CacheRequest) (*pb.CacheResponse, error) {
	in.Operation = pb.CacheRequest_INCREMENT
	return s.Call(ctx, in)
}

// Decrement decrements the value for the given key.
func (s *CacheServer) Decrement(ctx context.Context, in *pb.CacheRequest) (*pb.CacheResponse, error) {
	in.Operation = pb.CacheRequest_DECREMENT
	return s.Call(ctx, in)
}

// FlushAll deletes all key/value pairs from the cache.
func (s *CacheServer) FlushAll(ctx context.Context, in *pb.CacheRequest) (*pb.CacheResponse, error) {
	in.Operation = pb.CacheRequest_FLUSHALL
	return s.Call(ctx, in)
}
<commit_msg>comment for NewWithListener<commit_after>package server

import (
	"log"
	"net"
	"time"

	pb "github.com/joshrotenberg/grpc-cache/cache"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	"github.com/joshrotenberg/grpc-cache/lru"
)

// CacheServer encapsulates the cache server things.
type CacheServer struct {
	cache      *lru.Cache
	grpcServer *grpc.Server
	listener   net.Listener
}

// NewWithListener returns a new instance of the server given an initialized listener and
// maxEntries for the cache.
func NewWithListener(listener net.Listener, maxEntries int) *CacheServer {
	grpcServer := grpc.NewServer()
	server := CacheServer{
		cache:      lru.New(maxEntries),
		grpcServer: grpcServer,
		listener:   listener,
	}

	pb.RegisterCacheServer(grpcServer, &server)
	return &server
}

// New returns a new instance of the server. It takes host:port and maxEntries arguments,
// which defines the max number of entries allowed in the cache. Set to 0 for unlimited.
func New(address string, maxEntries int) (*CacheServer, error) {
	listener, err := net.Listen("tcp", address)
	if err != nil {
		return nil, err
	}
	return NewWithListener(listener, maxEntries), nil
}

// Start starts the cache server. It takes a host:port string on which to run the server.
func (s *CacheServer) Start() {
	go func() {
		if err := s.grpcServer.Serve(s.listener); err != nil {
			log.Fatalf("failed to serve: %v", err)
		}
	}()
}

// Stop tries to gracefull stop the server.
func (s *CacheServer) Stop() {
	s.grpcServer.GracefulStop()
}

func cacheError(err error, op pb.CacheRequest_Operation, key string) error {
	switch err {
	case lru.ErrNotFound:
		return status.Errorf(codes.NotFound, "%s error: '%s' not found", op, key)
	case lru.ErrExists:
		return status.Errorf(codes.AlreadyExists, "%s error: '%s' exists", op, key)
	}
	return err
}

func cacheResponse(err error, op pb.CacheRequest_Operation, item *pb.CacheItem) (*pb.CacheResponse, error) {
	if err != nil {
		return nil, cacheError(err, op, item.Key)
	}
	response := &pb.CacheResponse{
		Item: item,
	}
	return response, err
}

// Call calls the cache operation in in.Operation
func (s *CacheServer) Call(ctx context.Context, in *pb.CacheRequest) (*pb.CacheResponse, error) {

	var err error
	s.cache.Lock()
	defer s.cache.Unlock()

	switch in.Operation {
	case pb.CacheRequest_SET:
		s.cache.Set(in.Item.Key, in.Item.Value, time.Duration(in.Item.Ttl)*time.Second)
		return cacheResponse(nil, in.Operation, &pb.CacheItem{Key: in.Item.Key})
	case pb.CacheRequest_CAS:
		err = s.cache.Cas(in.Item.Key, in.Item.Value, time.Duration(in.Item.Ttl)*time.Second, uint64(in.Item.Cas))
		return cacheResponse(err, in.Operation, &pb.CacheItem{Key: in.Item.Key})
	case pb.CacheRequest_GET:
		value, err := s.cache.Get(in.Item.Key)
		return cacheResponse(err, in.Operation, &pb.CacheItem{Key: in.Item.Key, Value: value})
	case pb.CacheRequest_GETS:
		value, cas, err := s.cache.Gets(in.Item.Key)
		return cacheResponse(err, in.Operation, &pb.CacheItem{Key: in.Item.Key, Value: value, Cas: cas})
	case pb.CacheRequest_ADD:
		err = s.cache.Add(in.Item.Key, in.Item.Value, time.Duration(in.Item.Ttl)*time.Second)
		return cacheResponse(err, in.Operation, &pb.CacheItem{Key: in.Item.Key})
	case pb.CacheRequest_REPLACE:
		err = s.cache.Replace(in.Item.Key, in.Item.Value, time.Duration(in.Item.Ttl)*time.Second)
		return cacheResponse(err, in.Operation, &pb.CacheItem{Key: in.Item.Key})
	case pb.CacheRequest_DELETE:
		s.cache.Delete(in.Item.Key)
		return cacheResponse(nil, in.Operation, &pb.CacheItem{Key: in.Item.Key})
	case pb.CacheRequest_TOUCH:
		err = s.cache.Touch(in.Item.Key, time.Duration(in.Item.Ttl)*time.Second)
		return cacheResponse(err, in.Operation, &pb.CacheItem{Key: in.Item.Key})
	case pb.CacheRequest_APPEND:
		err = s.cache.Append(in.Item.Key, in.Append, time.Duration(in.Item.Ttl)*time.Second)
		return cacheResponse(err, in.Operation, &pb.CacheItem{Key: in.Item.Key})
	case pb.CacheRequest_PREPEND:
		err = s.cache.Prepend(in.Item.Key, in.Prepend, time.Duration(in.Item.Ttl)*time.Second)
		return cacheResponse(err, in.Operation, &pb.CacheItem{Key: in.Item.Key})
	case pb.CacheRequest_INCREMENT:
		err = s.cache.Increment(in.Item.Key, in.Increment)
		return cacheResponse(err, in.Operation, &pb.CacheItem{Key: in.Item.Key})
	case pb.CacheRequest_DECREMENT:
		err = s.cache.Decrement(in.Item.Key, in.Decrement)
		return cacheResponse(err, in.Operation, &pb.CacheItem{Key: in.Item.Key})
	case pb.CacheRequest_FLUSHALL:
		s.cache.FlushAll()
		return cacheResponse(nil, in.Operation, nil)
	default:
		return nil, status.Errorf(codes.Unimplemented, "unrecognized cache command %d", in.Operation)
	}

}

// Set stores a key/value pair in the cache.
func (s *CacheServer) Set(ctx context.Context, in *pb.CacheRequest) (*pb.CacheResponse, error) {
	in.Operation = pb.CacheRequest_SET
	return s.Call(ctx, in)
}

// Cas does a "check and set" or "compare and swap" operation. If the given key exists
// and the currently stored CAS value matches the one supplied, the value is stored.
func (s *CacheServer) Cas(ctx context.Context, in *pb.CacheRequest) (*pb.CacheResponse, error) {
	in.Operation = pb.CacheRequest_CAS
	return s.Call(ctx, in)
}

// Get gets a key/value pair from the cache.
func (s *CacheServer) Get(ctx context.Context, in *pb.CacheRequest) (*pb.CacheResponse, error) {
	in.Operation = pb.CacheRequest_GET
	return s.Call(ctx, in)
}

// Gets gets a key/value pair from the cache and includes its CAS value.
func (s *CacheServer) Gets(ctx context.Context, in *pb.CacheRequest) (*pb.CacheResponse, error) {
	in.Operation = pb.CacheRequest_GETS
	return s.Call(ctx, in)
}

// Add adds a key/value pair to the cache.
func (s *CacheServer) Add(ctx context.Context, in *pb.CacheRequest) (*pb.CacheResponse, error) {
	in.Operation = pb.CacheRequest_ADD
	return s.Call(ctx, in)
}

// Replace replaces a key/value pair in the cache.
func (s *CacheServer) Replace(ctx context.Context, in *pb.CacheRequest) (*pb.CacheResponse, error) {
	in.Operation = pb.CacheRequest_REPLACE
	return s.Call(ctx, in)
}

// Delete deletes a key/value pair from the cache.
func (s *CacheServer) Delete(ctx context.Context, in *pb.CacheRequest) (*pb.CacheResponse, error) {
	in.Operation = pb.CacheRequest_DELETE
	return s.Call(ctx, in)
}

// Touch updates the key/value pair's cas and ttl.
func (s *CacheServer) Touch(ctx context.Context, in *pb.CacheRequest) (*pb.CacheResponse, error) {
	in.Operation = pb.CacheRequest_TOUCH
	return s.Call(ctx, in)
}

// Append appends the value to the current value for the key.
func (s *CacheServer) Append(ctx context.Context, in *pb.CacheRequest) (*pb.CacheResponse, error) {
	in.Operation = pb.CacheRequest_APPEND
	return s.Call(ctx, in)
}

// Prepend prepends the value to the current value for the key.
func (s *CacheServer) Prepend(ctx context.Context, in *pb.CacheRequest) (*pb.CacheResponse, error) {
	in.Operation = pb.CacheRequest_PREPEND
	return s.Call(ctx, in)
}

// Increment increments the value for key.
func (s *CacheServer) Increment(ctx context.Context, in *pb.CacheRequest) (*pb.CacheResponse, error) {
	in.Operation = pb.CacheRequest_INCREMENT
	return s.Call(ctx, in)
}

// Decrement decrements the value for the given key.
func (s *CacheServer) Decrement(ctx context.Context, in *pb.CacheRequest) (*pb.CacheResponse, error) {
	in.Operation = pb.CacheRequest_DECREMENT
	return s.Call(ctx, in)
}

// FlushAll deletes all key/value pairs from the cache.
func (s *CacheServer) FlushAll(ctx context.Context, in *pb.CacheRequest) (*pb.CacheResponse, error) {
	in.Operation = pb.CacheRequest_FLUSHALL
	return s.Call(ctx, in)
}
<|endoftext|>


<commit_before>package quiz

import (
	"encoding/json"
	"net/http"
	"strconv"
	"time"

	"github.com/dgraph-io/gru/admin/server"
	"github.com/dgraph-io/gru/dgraph"
	"github.com/dgraph-io/gru/x"
)

type Tag struct {
	Name string `json:"name"`
}

// Question is marshalled to JSON and sent to the client.
type Question struct {
	Uid string `json:"uid"`

	// cuid represents the uid of the question asked to the candidate, it is linked
	// to the original question uid.
	Cid     string   `json:"cuid"`
	Text    string   `json:"text"`
	Options []Answer `json:"question.option"`
	Tags    []Tag    `json:"question.tag"`
	// TODO - Remove the ,string after we incorporate Dgraph schema here.
	IsMultiple bool    `json:"multiple"`
	Positive   float64 `json:"negative"`
	Negative   float64 `json:"positive"`
	// Score of the candidate is sent as part of the questions API.
	Score     float64 `json:"score"`
	TimeTaken string  `json:"time_taken"`
	// Current question number.
	Idx int `json:"idx"`
	// Total number of questions.
	NumQns int `json:"num_qns"`
}

func QuestionHandler(w http.ResponseWriter, r *http.Request) {
	sr := server.Response{}
	userId, err := validateToken(r)
	if err != nil {
		sr.Write(w, err.Error(), "Unauthorized", http.StatusUnauthorized)
		return
	}

	c, err := readMap(userId)
	if err != nil {
		sr.Write(w, "", "Candidate not found.", http.StatusBadRequest)
		return
	}

	if timeLeft(c.quizStart, c.quizDuration) < 0 {
		sr.Write(w, "", "Your quiz has already finished.", http.StatusBadRequest)
		return
	}

	if c.quizStart.IsZero() {
		// This means its the first question he is being asked. Lets
		// store quizStart so that we can use to calculate timeLeft for
		// Ping API. Lets also persist it to database, so that we can
		// recover it incase we crash.
		c.quizStart = time.Now().UTC()
		updateMap(userId, c)
		m := new(dgraph.Mutation)
		m.SetString(userId, "quiz_start", c.quizStart.Format(timeLayout))
		_, err := dgraph.SendMutation(m)
		if err != nil {
			sr.Write(w, "", err.Error(), http.StatusInternalServerError)
			return
		}
	}

	if len(c.qns[c.level]) == 0 || c.score <= c.quizThreshold {
		// No more questions to ask or score is less than threshold.
		// Client ends quiz when question id is END.
		q := Question{
			Uid:    "END",
			Score: x.ToFixed(c.score, 2),
		}

		// Lets store that the user successfully completed the test.
		m := new(dgraph.Mutation)
		m.Set(`<` + userId + `> <complete> "true" .`)
		// Completed at is used to reject candidates whose score is < cutoff
		m.Set(`<` + userId + `> <completed_at> "` + time.Now().UTC().Format(timeLayout) + `" .`)
		m.Set(`<` + userId + `> <score> "` + strconv.FormatFloat(x.ToFixed(c.score, 2), 'g', -1, 64) + `" .`)
		_, err := dgraph.SendMutation(m)
		if err != nil {
			sr.Write(w, "", err.Error(), http.StatusInternalServerError)
			return
		}

		b, err := json.Marshal(q)
		if err != nil {
			sr.Write(w, err.Error(), "", http.StatusInternalServerError)
			return
		}
		if err = sendMail(c, userId); err != nil {
			sr.Write(w, err.Error(), "", http.StatusInternalServerError)
			return
		}
		w.Write(b)
		return
	}

	qn := c.qns[c.level][0]
	if c.lastQnAsked.IsZero() {
		qn.TimeTaken = "0s"
		c.lastQnAsked = time.Now().UTC()
		updateMap(userId, c)
	} else {
		qn.TimeTaken = time.Now().UTC().Sub(c.lastQnAsked).String()
	}

	qn.Score = x.ToFixed(c.score, 2)
	shuffleOptions(qn.Options)

	qn.NumQns = c.numQuestions
	qn.Idx = c.qnIdx
	updateMap(userId, c)
	if c.lastQnUid != "" && c.lastQnUid == qn.Uid {
		qn.Cid = c.lastQnCuid
		server.MarshalAndWrite(w, &qn)
		return
	}

	m := new(dgraph.Mutation)
	m.Set(`<` + userId + `> <candidate.question> <_:qn> .`)
	m.Set(`<_:qn> <question.uid> <` + qn.Uid + `> .`)
	m.Set(`<` + qn.Uid + `> <question.candidate> <` + userId + `> .`)
	m.Set(`<_:qn> <question.asked> "` + time.Now().UTC().Format("2006-01-02T15:04:05Z07:00") + `" .`)
	m.Set(`<` + userId + `> <candidate.lastqnuid> "` + qn.Uid + `" .`)
	res, err := dgraph.SendMutation(m)
	if err != nil {
		sr.Write(w, "", err.Error(), http.StatusInternalServerError)
		return
	}
	if res.Uids["qn"] == "" {
		sr.Write(w, res.Message, "", http.StatusInternalServerError)
		return
	}

	c.lastQnCuid = res.Uids["qn"]
	qn.Cid = res.Uids["qn"]
	c.lastQnUid = qn.Uid
	qn.Idx = c.qnIdx + 1
	c.qnIdx += 1
	updateMap(userId, c)
	server.MarshalAndWrite(w, &qn)
}
<commit_msg>TODO: write tests<commit_after>package quiz

import (
	"encoding/json"
	"net/http"
	"strconv"
	"time"

	"github.com/dgraph-io/gru/admin/server"
	"github.com/dgraph-io/gru/dgraph"
	"github.com/dgraph-io/gru/x"
)

type Tag struct {
	Name string `json:"name"`
}

// Question is marshalled to JSON and sent to the client.
type Question struct {
	Uid string `json:"uid"`

	// TODO - I wanted to write a sarcastic comment about this name choice,
	// but all my ideas were totally passive agressive
	// Rename cuid to something that makes sense without reading the below comment

	// cuid represents the uid of the question asked to the candidate, it is linked
	// to the original question uid.
	Cid     string   `json:"cuid"`
	Text    string   `json:"text"`
	Options []Answer `json:"question.option"`
	Tags    []Tag    `json:"question.tag"`
	// TODO - Remove the ,string after we incorporate Dgraph schema here.
	IsMultiple bool    `json:"multiple"`
	Positive   float64 `json:"positive"`
	Negative   float64 `json:"negative"`
	// Score of the candidate is sent as part of the questions API.
	Score     float64 `json:"score"`
	TimeTaken string  `json:"time_taken"`
	// Current question number.
	Idx int `json:"idx"`
	// Total number of questions.
	NumQns int `json:"num_qns"`
}

func QuestionHandler(w http.ResponseWriter, r *http.Request) {
	sr := server.Response{}
	userId, err := validateToken(r)
	if err != nil {
		sr.Write(w, err.Error(), "Unauthorized", http.StatusUnauthorized)
		return
	}

	c, err := readMap(userId)
	if err != nil {
		sr.Write(w, "", "Candidate not found.", http.StatusBadRequest)
		return
	}

	if timeLeft(c.quizStart, c.quizDuration) < 0 {
		sr.Write(w, "", "Your quiz has already finished.", http.StatusBadRequest)
		return
	}

	if c.quizStart.IsZero() {
		// This means its the first question he is being asked. Lets
		// store quizStart so that we can use to calculate timeLeft for
		// Ping API. Lets also persist it to database, so that we can
		// recover it incase we crash.
		c.quizStart = time.Now().UTC()
		updateMap(userId, c)
		m := new(dgraph.Mutation)
		m.SetString(userId, "quiz_start", c.quizStart.Format(timeLayout))
		_, err := dgraph.SendMutation(m)
		if err != nil {
			sr.Write(w, "", err.Error(), http.StatusInternalServerError)
			return
		}
	}

	if len(c.qns[c.level]) == 0 || c.score <= c.quizThreshold {
		// No more questions to ask or score is less than threshold.
		// Client ends quiz when question id is END.
		q := Question{
			Uid:    "END",
			Score: x.ToFixed(c.score, 2),
		}

		// Lets store that the user successfully completed the test.
		m := new(dgraph.Mutation)
		m.Set(`<` + userId + `> <complete> "true" .`)
		// Completed at is used to reject candidates whose score is < cutoff
		m.Set(`<` + userId + `> <completed_at> "` + time.Now().UTC().Format(timeLayout) + `" .`)
		m.Set(`<` + userId + `> <score> "` + strconv.FormatFloat(x.ToFixed(c.score, 2), 'g', -1, 64) + `" .`)
		_, err := dgraph.SendMutation(m)
		if err != nil {
			sr.Write(w, "", err.Error(), http.StatusInternalServerError)
			return
		}

		b, err := json.Marshal(q)
		if err != nil {
			sr.Write(w, err.Error(), "", http.StatusInternalServerError)
			return
		}
		if err = sendMail(c, userId); err != nil {
			sr.Write(w, err.Error(), "", http.StatusInternalServerError)
			return
		}
		w.Write(b)
		return
	}

	qn := c.qns[c.level][0]
	if c.lastQnAsked.IsZero() {
		qn.TimeTaken = "0s"
		c.lastQnAsked = time.Now().UTC()
		updateMap(userId, c)
	} else {
		qn.TimeTaken = time.Now().UTC().Sub(c.lastQnAsked).String()
	}

	qn.Score = x.ToFixed(c.score, 2)
	shuffleOptions(qn.Options)

	qn.NumQns = c.numQuestions
	qn.Idx = c.qnIdx
	updateMap(userId, c)
	if c.lastQnUid != "" && c.lastQnUid == qn.Uid {
		qn.Cid = c.lastQnCuid
		server.MarshalAndWrite(w, &qn)
		return
	}

	m := new(dgraph.Mutation)
	m.Set(`<` + userId + `> <candidate.question> <_:qn> .`)
	m.Set(`<_:qn> <question.uid> <` + qn.Uid + `> .`)
	m.Set(`<` + qn.Uid + `> <question.candidate> <` + userId + `> .`)
	m.Set(`<_:qn> <question.asked> "` + time.Now().UTC().Format("2006-01-02T15:04:05Z07:00") + `" .`)
	m.Set(`<` + userId + `> <candidate.lastqnuid> "` + qn.Uid + `" .`)
	res, err := dgraph.SendMutation(m)
	if err != nil {
		sr.Write(w, "", err.Error(), http.StatusInternalServerError)
		return
	}
	if res.Uids["qn"] == "" {
		sr.Write(w, res.Message, "", http.StatusInternalServerError)
		return
	}

	c.lastQnCuid = res.Uids["qn"]
	qn.Cid = res.Uids["qn"]
	c.lastQnUid = qn.Uid
	qn.Idx = c.qnIdx + 1
	c.qnIdx += 1
	updateMap(userId, c)
	server.MarshalAndWrite(w, &qn)
}
<|endoftext|>


<commit_before>package marathon

import (
	"encoding/json"
	"github.com/QubitProducts/bamboo/configuration"
	"io/ioutil"
	"net/http"
	"sort"
	"strings"
)

// Describes an app process running
type Task struct {
	Host  string
	Port  int
	Ports []int
}

// A health check on the application
type HealthCheck struct {
	// One of TCP, HTTP or COMMAND
	Protocol string
	// The path (if Protocol is HTTP)
	Path string
	// The position of the port targeted in the ports array
	PortIndex int
}

// An app may have multiple processes
type App struct {
	Id              string
	EscapedId       string
	HealthCheckPath string
	HealthChecks    []HealthCheck
	Tasks           []Task
	ServicePort     int
	ServicePorts    []int
	Env             map[string]string
	Labels          map[string]string
}

type AppList []App

func (slice AppList) Len() int {
	return len(slice)
}

func (slice AppList) Less(i, j int) bool {
	return slice[i].Id < slice[j].Id
}

func (slice AppList) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

type marathonTaskList []marathonTask

type marathonTasks struct {
	Tasks marathonTaskList `json:"tasks"`
}

type marathonTask struct {
	AppId        string
	Id           string
	Host         string
	Ports        []int
	ServicePorts []int
	StartedAt    string
	StagedAt     string
	Version      string
}

func (slice marathonTaskList) Len() int {
	return len(slice)
}

func (slice marathonTaskList) Less(i, j int) bool {
	return slice[i].Id < slice[j].Id
}

func (slice marathonTaskList) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

type marathonApps struct {
	Apps []marathonApp `json:"apps"`
}

type marathonApp struct {
	Id           string                `json:"id"`
	HealthChecks []marathonHealthCheck `json:"healthChecks"`
	Ports        []int                 `json:"ports"`
	Env          map[string]string     `json:"env"`
	Labels       map[string]string     `json:"labels"`
}

type marathonHealthCheck struct {
	Path      string `json:"path"`
	Protocol  string `json:"protocol"`
	PortIndex int    `json:"portIndex"`
}

func fetchMarathonApps(endpoint string, conf *configuration.Configuration) (map[string]marathonApp, error) {
	client := &http.Client{}
	req, _ := http.NewRequest("GET", endpoint+"/v2/apps", nil)
	req.Header.Add("Accept", "application/json")
	req.Header.Add("Content-Type", "application/json")
	if len(conf.Marathon.User) > 0 && len(conf.Marathon.Password) > 0 {
		req.SetBasicAuth(conf.Marathon.User, conf.Marathon.Password)
	}
	response, err := client.Do(req)

	if err != nil {
		return nil, err
	}

	defer response.Body.Close()
	var appResponse marathonApps

	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(contents, &appResponse)
	if err != nil {
		return nil, err
	}

	dataById := map[string]marathonApp{}

	for _, appConfig := range appResponse.Apps {
		dataById[appConfig.Id] = appConfig
	}

	return dataById, nil
}

func fetchTasks(endpoint string, conf *configuration.Configuration) (map[string]marathonTaskList, error) {
	client := &http.Client{}
	req, _ := http.NewRequest("GET", endpoint+"/v2/tasks", nil)
	req.Header.Add("Accept", "application/json")
	req.Header.Add("Content-Type", "application/json")
	if len(conf.Marathon.User) > 0 && len(conf.Marathon.Password) > 0 {
		req.SetBasicAuth(conf.Marathon.User, conf.Marathon.Password)
	}
	response, err := client.Do(req)

	var tasks marathonTasks

	if err != nil {
		return nil, err
	}

	contents, err := ioutil.ReadAll(response.Body)
	defer response.Body.Close()
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(contents, &tasks)
	if err != nil {
		return nil, err
	}

	taskList := tasks.Tasks
	sort.Sort(taskList)

	tasksById := map[string]marathonTaskList{}
	for _, task := range taskList {
		if tasksById[task.AppId] == nil {
			tasksById[task.AppId] = marathonTaskList{}
		}
		tasksById[task.AppId] = append(tasksById[task.AppId], task)
	}

	for _, task_list := range tasksById {
		sort.Sort(task_list)
	}

	return tasksById, nil
}

func createApps(tasksById map[string]marathonTaskList, marathonApps map[string]marathonApp) AppList {

	apps := AppList{}

	for appId, mApp := range marathonApps {

		// Try to handle old app id format without slashes
		appPath := appId
		if !strings.HasPrefix(appId, "/") {
			appPath = "/" + appId
		}

		// build App from marathonApp
		app := App{
			Id:              appPath,
			EscapedId:       strings.Replace(appId, "/", "::", -1),
			HealthCheckPath: parseHealthCheckPath(mApp.HealthChecks),
			Env:             mApp.Env,
			Labels:          mApp.Labels,
		}

		app.HealthChecks = make([]HealthCheck, 0, len(mApp.HealthChecks))
		for _, marathonCheck := range mApp.HealthChecks {
			check := HealthCheck{
				Protocol:  marathonCheck.Protocol,
				Path:      marathonCheck.Path,
				PortIndex: marathonCheck.PortIndex,
			}
			app.HealthChecks = append(app.HealthChecks, check)
		}

		if len(mApp.Ports) > 0 {
			app.ServicePort = mApp.Ports[0]
			app.ServicePorts = mApp.Ports
		}

		// build Tasks for this App
		tasks := []Task{}
		for _, mTask := range tasksById[appId] {
			if len(mTask.Ports) > 0 {
				t := Task{
					Host:  mTask.Host,
					Port:  mTask.Ports[0],
					Ports: mTask.Ports,
				}
				tasks = append(tasks, t)
			}
		}
		app.Tasks = tasks

		apps = append(apps, app)
	}
	return apps
}

func parseHealthCheckPath(checks []marathonHealthCheck) string {
	for _, check := range checks {
		if check.Protocol != "HTTP" {
			continue
		}
		return check.Path
	}
	return ""
}

/*
	Apps returns a struct that describes Marathon current app and their
	sub tasks information.

	Parameters:
		endpoint: Marathon HTTP endpoint, e.g. http://localhost:8080
*/
func FetchApps(maraconf configuration.Marathon, conf *configuration.Configuration) (AppList, error) {

	var applist AppList
	var err error

	// try all configured endpoints until one succeeds
	for _, url := range maraconf.Endpoints() {
		applist, err = _fetchApps(url, conf)
		if err == nil {
			return applist, err
		}
	}
	// return last error
	return nil, err
}

func _fetchApps(url string, conf *configuration.Configuration) (AppList, error) {
	tasks, err := fetchTasks(url, conf)
	if err != nil {
		return nil, err
	}

	marathonApps, err := fetchMarathonApps(url, conf)
	if err != nil {
		return nil, err
	}

	apps := createApps(tasks, marathonApps)
	sort.Sort(apps)
	return apps, nil
}
<commit_msg>Adding a new marathon App struct field for the mesos dns name: the marathon group names are reversed and appended to appname.  '/a/b/app' becomes app-b-a as a service name<commit_after>package marathon

import (
	"encoding/json"
	"github.com/QubitProducts/bamboo/configuration"
	"io/ioutil"
	"net/http"
	"sort"
	"strings"
)

// Describes an app process running
type Task struct {
	Host  string
	Port  int
	Ports []int
}

// A health check on the application
type HealthCheck struct {
	// One of TCP, HTTP or COMMAND
	Protocol string
	// The path (if Protocol is HTTP)
	Path string
	// The position of the port targeted in the ports array
	PortIndex int
}

// An app may have multiple processes
type App struct {
	Id              string
	MesosDnsId      string
	EscapedId       string
	HealthCheckPath string
	HealthChecks    []HealthCheck
	Tasks           []Task
	ServicePort     int
	ServicePorts    []int
	Env             map[string]string
	Labels          map[string]string
}

type AppList []App

func (slice AppList) Len() int {
	return len(slice)
}

func (slice AppList) Less(i, j int) bool {
	return slice[i].Id < slice[j].Id
}

func (slice AppList) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

type marathonTaskList []marathonTask

type marathonTasks struct {
	Tasks marathonTaskList `json:"tasks"`
}

type marathonTask struct {
	AppId        string
	Id           string
	Host         string
	Ports        []int
	ServicePorts []int
	StartedAt    string
	StagedAt     string
	Version      string
}

func (slice marathonTaskList) Len() int {
	return len(slice)
}

func (slice marathonTaskList) Less(i, j int) bool {
	return slice[i].Id < slice[j].Id
}

func (slice marathonTaskList) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

type marathonApps struct {
	Apps []marathonApp `json:"apps"`
}

type marathonApp struct {
	Id           string                `json:"id"`
	HealthChecks []marathonHealthCheck `json:"healthChecks"`
	Ports        []int                 `json:"ports"`
	Env          map[string]string     `json:"env"`
	Labels       map[string]string     `json:"labels"`
}

type marathonHealthCheck struct {
	Path      string `json:"path"`
	Protocol  string `json:"protocol"`
	PortIndex int    `json:"portIndex"`
}

func fetchMarathonApps(endpoint string, conf *configuration.Configuration) (map[string]marathonApp, error) {
	client := &http.Client{}
	req, _ := http.NewRequest("GET", endpoint+"/v2/apps", nil)
	req.Header.Add("Accept", "application/json")
	req.Header.Add("Content-Type", "application/json")
	if len(conf.Marathon.User) > 0 && len(conf.Marathon.Password) > 0 {
		req.SetBasicAuth(conf.Marathon.User, conf.Marathon.Password)
	}
	response, err := client.Do(req)

	if err != nil {
		return nil, err
	}

	defer response.Body.Close()
	var appResponse marathonApps

	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(contents, &appResponse)
	if err != nil {
		return nil, err
	}

	dataById := map[string]marathonApp{}

	for _, appConfig := range appResponse.Apps {
		dataById[appConfig.Id] = appConfig
	}

	return dataById, nil
}

func fetchTasks(endpoint string, conf *configuration.Configuration) (map[string]marathonTaskList, error) {
	client := &http.Client{}
	req, _ := http.NewRequest("GET", endpoint+"/v2/tasks", nil)
	req.Header.Add("Accept", "application/json")
	req.Header.Add("Content-Type", "application/json")
	if len(conf.Marathon.User) > 0 && len(conf.Marathon.Password) > 0 {
		req.SetBasicAuth(conf.Marathon.User, conf.Marathon.Password)
	}
	response, err := client.Do(req)

	var tasks marathonTasks

	if err != nil {
		return nil, err
	}

	contents, err := ioutil.ReadAll(response.Body)
	defer response.Body.Close()
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(contents, &tasks)
	if err != nil {
		return nil, err
	}

	taskList := tasks.Tasks
	sort.Sort(taskList)

	tasksById := map[string]marathonTaskList{}
	for _, task := range taskList {
		if tasksById[task.AppId] == nil {
			tasksById[task.AppId] = marathonTaskList{}
		}
		tasksById[task.AppId] = append(tasksById[task.AppId], task)
	}

	for _, task_list := range tasksById {
		sort.Sort(task_list)
	}

	return tasksById, nil
}

func createApps(tasksById map[string]marathonTaskList, marathonApps map[string]marathonApp) AppList {

	apps := AppList{}

	for appId, mApp := range marathonApps {

		// Try to handle old app id format without slashes
		appPath := appId
		if !strings.HasPrefix(appId, "/") {
			appPath = "/" + appId
		}

		// split up groups and recombine for how mesos-dns/consul/etc use service name
		//   "/nested/group/app" -> "app-group-nested"
		groups := strings.Split(appPath, ",")
		reverseGroups := []string{}
		for i := len(groups) - 1; i >= 0; i-- {
			if groups[i] != "" {
				reverseGroups = append(reverseGroups, groups[i])
			}
		}
		mesosId := strings.Join(reverseGroups, "-")

		// build App from marathonApp
		app := App{
			Id:              appPath,
			MesosDnsId:      mesosId,
			EscapedId:       strings.Replace(appId, "/", "::", -1),
			HealthCheckPath: parseHealthCheckPath(mApp.HealthChecks),
			Env:             mApp.Env,
			Labels:          mApp.Labels,
		}

		app.HealthChecks = make([]HealthCheck, 0, len(mApp.HealthChecks))
		for _, marathonCheck := range mApp.HealthChecks {
			check := HealthCheck{
				Protocol:  marathonCheck.Protocol,
				Path:      marathonCheck.Path,
				PortIndex: marathonCheck.PortIndex,
			}
			app.HealthChecks = append(app.HealthChecks, check)
		}

		if len(mApp.Ports) > 0 {
			app.ServicePort = mApp.Ports[0]
			app.ServicePorts = mApp.Ports
		}

		// build Tasks for this App
		tasks := []Task{}
		for _, mTask := range tasksById[appId] {
			if len(mTask.Ports) > 0 {
				t := Task{
					Host:  mTask.Host,
					Port:  mTask.Ports[0],
					Ports: mTask.Ports,
				}
				tasks = append(tasks, t)
			}
		}
		app.Tasks = tasks

		apps = append(apps, app)
	}
	return apps
}

func parseHealthCheckPath(checks []marathonHealthCheck) string {
	for _, check := range checks {
		if check.Protocol != "HTTP" {
			continue
		}
		return check.Path
	}
	return ""
}

/*
	Apps returns a struct that describes Marathon current app and their
	sub tasks information.

	Parameters:
		endpoint: Marathon HTTP endpoint, e.g. http://localhost:8080
*/
func FetchApps(maraconf configuration.Marathon, conf *configuration.Configuration) (AppList, error) {

	var applist AppList
	var err error

	// try all configured endpoints until one succeeds
	for _, url := range maraconf.Endpoints() {
		applist, err = _fetchApps(url, conf)
		if err == nil {
			return applist, err
		}
	}
	// return last error
	return nil, err
}

func _fetchApps(url string, conf *configuration.Configuration) (AppList, error) {
	tasks, err := fetchTasks(url, conf)
	if err != nil {
		return nil, err
	}

	marathonApps, err := fetchMarathonApps(url, conf)
	if err != nil {
		return nil, err
	}

	apps := createApps(tasks, marathonApps)
	sort.Sort(apps)
	return apps, nil
}
<|endoftext|>