<commit_before>package server

import (
	"context"
	"fmt"
	"path"
	"time"

	log "github.com/sirupsen/logrus"
	"k8s.io/kubernetes/pkg/api"
	labelspkg "k8s.io/kubernetes/pkg/labels"
	kube_watch "k8s.io/kubernetes/pkg/watch"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/backoff"
	col "github.com/pachyderm/pachyderm/src/server/pkg/collection"
	"github.com/pachyderm/pachyderm/src/server/pkg/dlock"
	"github.com/pachyderm/pachyderm/src/server/pkg/util"
	"github.com/pachyderm/pachyderm/src/server/pkg/watch"
	ppsserver "github.com/pachyderm/pachyderm/src/server/pps"
)

const (
	masterLockPath = "_master_lock"
)

var (
	failures = map[string]bool{
		"InvalidImageName": true,
		"ErrImagePull":     true,
	}
)

// The master process is responsible for creating/deleting workers as
// pipelines are created/removed.
func (a *apiServer) master() {
	masterLock := dlock.NewDLock(a.etcdClient, path.Join(a.etcdPrefix, masterLockPath))
	backoff.RetryNotify(func() error {
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()

		ctx, err := masterLock.Lock(ctx)
		if err != nil {
			return err
		}
		defer masterLock.Unlock(ctx)

		log.Infof("Launching PPS master process")

		pipelineWatcher, err := a.pipelines.ReadOnly(ctx).WatchWithPrev()
		if err != nil {
			return fmt.Errorf("error creating watch: %+v", err)
		}
		defer pipelineWatcher.Close()

		// watchChan will be nil if the Watch call below errors, this means
		// that we won't receive events from k8s and won't be able to detect
		// errors in pods. We could just return that error and retry but that
		// prevents pachyderm from creating pipelines when there's an issue
		// talking to k8s.
		var watchChan <-chan kube_watch.Event
		kubePipelineWatch, err := a.kubeClient.Pods(a.namespace).Watch(api.ListOptions{
			LabelSelector: labelspkg.SelectorFromSet(map[string]string{
				"component": "worker",
			}),
			Watch: true,
		})
		if err != nil {
			log.Errorf("failed to watch kuburnetes pods: %v", err)
		} else {
			watchChan = kubePipelineWatch.ResultChan()
			defer kubePipelineWatch.Stop()
		}

		for {
			select {
			case event := <-pipelineWatcher.Watch():
				if event.Err != nil {
					return fmt.Errorf("event err: %+v", event.Err)
				}
				switch event.Type {
				case watch.EventPut:
					var pipelineName string
					var pipelineInfo pps.PipelineInfo
					if err := event.Unmarshal(&pipelineName, &pipelineInfo); err != nil {
						return err
					}

					if pipelineInfo.Salt == "" {
						if _, err := col.NewSTM(ctx, a.etcdClient, func(stm col.STM) error {
							pipelines := a.pipelines.ReadWrite(stm)
							newPipelineInfo := new(pps.PipelineInfo)
							if err := pipelines.Get(pipelineInfo.Pipeline.Name, newPipelineInfo); err != nil {
								return fmt.Errorf("error getting pipeline %s: %+v", pipelineName, err)
							}
							if newPipelineInfo.Salt == "" {
								newPipelineInfo.Salt = uuid.NewWithoutDashes()
							}
							pipelines.Put(pipelineInfo.Pipeline.Name, newPipelineInfo)
							return nil
						}); err != nil {
							return err
						}
					}

					var prevPipelineInfo pps.PipelineInfo
					if event.PrevKey != nil {
						if err := event.UnmarshalPrev(&pipelineName, &prevPipelineInfo); err != nil {
							return err
						}
					}

					// If the pipeline has been stopped, delete workers
					if pipelineStateToStopped(pipelineInfo.State) {
						log.Infof("master: deleting workers for pipeline %s", pipelineInfo.Pipeline.Name)
						if err := a.deleteWorkersForPipeline(&pipelineInfo); err != nil {
							return err
						}
					}

					// If the pipeline has been restarted, create workers
					if !pipelineStateToStopped(pipelineInfo.State) && event.PrevKey != nil && pipelineStateToStopped(prevPipelineInfo.State) {
						if err := a.upsertWorkersForPipeline(&pipelineInfo); err != nil {
							if err := a.setPipelineFailure(ctx, pipelineInfo.Pipeline.Name, fmt.Sprintf("failed to create workers: %s", err.Error())); err != nil {
								return err
							}
							continue
						}
					}

					// If the pipeline has been updated, create new workers
					if pipelineInfo.Version > prevPipelineInfo.Version && !pipelineStateToStopped(pipelineInfo.State) {
						log.Infof("master: creating/updating workers for pipeline %s", pipelineInfo.Pipeline.Name)
						if event.PrevKey != nil {
							if err := a.deleteWorkersForPipeline(&prevPipelineInfo); err != nil {
								return err
							}
						}
						if err := a.upsertWorkersForPipeline(&pipelineInfo); err != nil {
							if err := a.setPipelineFailure(ctx, pipelineInfo.Pipeline.Name, fmt.Sprintf("failed to create workers: %s", err.Error())); err != nil {
								return err
							}
							continue
						}
					}
				case watch.EventDelete:
					var pipelineName string
					var pipelineInfo pps.PipelineInfo
					if err := event.UnmarshalPrev(&pipelineName, &pipelineInfo); err != nil {
						return err
					}
					if err := a.deleteWorkersForPipeline(&pipelineInfo); err != nil {
						return err
					}
				}
			case event := <-watchChan:
				// if we get an error we restart the watch, k8s watches seem to
				// sometimes get stuck in a loop returning events with Type =
				// "" we treat these as errors since otherwise we get an
				// endless stream of them and can't do anything.
				if event.Type == kube_watch.Error || event.Type == "" {
					kubePipelineWatch.Stop()
					kubePipelineWatch, err = a.kubeClient.Pods(a.namespace).Watch(api.ListOptions{
						LabelSelector: labelspkg.SelectorFromSet(map[string]string{
							"component": "worker",
						}),
						Watch: true,
					})
					if err != nil {
						return err
					}
					defer kubePipelineWatch.Stop()
				}
				pod, ok := event.Object.(*api.Pod)
				if !ok {
					continue
				}
				if pod.Status.Phase == api.PodFailed {
					log.Errorf("pod failed because: %s", pod.Status.Message)
				}
				for _, status := range pod.Status.ContainerStatuses {
					if status.Name == "user" && status.State.Waiting != nil && failures[status.State.Waiting.Reason] {
						if err := a.setPipelineFailure(ctx, pod.ObjectMeta.Annotations["pipelineName"], status.State.Waiting.Message); err != nil {
							return err
						}
					}
				}
			}
		}
	}, backoff.NewInfiniteBackOff(), func(err error, d time.Duration) error {
		log.Errorf("master: error running the master process: %v; retrying in %v", err, d)
		return nil
	})
}

func (a *apiServer) setPipelineFailure(ctx context.Context, pipelineName string, reason string) error {
	// Set pipeline state to failure
	_, err := col.NewSTM(ctx, a.etcdClient, func(stm col.STM) error {
		pipelines := a.pipelines.ReadWrite(stm)
		pipelineInfo := new(pps.PipelineInfo)
		if err := pipelines.Get(pipelineName, pipelineInfo); err != nil {
			return err
		}
		pipelineInfo.State = pps.PipelineState_PIPELINE_FAILURE
		pipelineInfo.Reason = reason
		pipelines.Put(pipelineName, pipelineInfo)
		return nil
	})
	return err
}

func (a *apiServer) upsertWorkersForPipeline(pipelineInfo *pps.PipelineInfo) error {
	var errCount int
	return backoff.RetryNotify(func() error {
		parallelism, err := ppsserver.GetExpectedNumWorkers(a.kubeClient, pipelineInfo.ParallelismSpec)
		if err != nil {
			return err
		}
		var resources *api.ResourceList
		if pipelineInfo.ResourceSpec != nil {
			resources, err = util.GetResourceListFromPipeline(pipelineInfo)
			if err != nil {
				return err
			}
		}

		// Retrieve the current state of the RC.  If the RC is scaled down,
		// we want to ensure that it remains scaled down.
		rc := a.kubeClient.ReplicationControllers(a.namespace)
		workerRc, err := rc.Get(ppsserver.PipelineRcName(pipelineInfo.Pipeline.Name, pipelineInfo.Version))
		if err == nil {
			if (workerRc.Spec.Template.Spec.Containers[0].Resources.Requests == nil) && workerRc.Spec.Replicas == 1 {
				parallelism = 1
				resources = nil
			}
		}

		options := a.getWorkerOptions(
			pipelineInfo.Pipeline.Name,
			ppsserver.PipelineRcName(pipelineInfo.Pipeline.Name, pipelineInfo.Version),
			int32(parallelism),
			resources,
			pipelineInfo.Transform,
			pipelineInfo.CacheSize,
			pipelineInfo.Service)
		// Set the pipeline name env
		options.workerEnv = append(options.workerEnv, api.EnvVar{
			Name:  client.PPSPipelineNameEnv,
			Value: pipelineInfo.Pipeline.Name,
		})
		return a.createWorkerRc(options)
	}, backoff.NewInfiniteBackOff(), func(err error, d time.Duration) error {
		errCount++
		if errCount >= 3 {
			return err
		}
		log.Errorf("error creating workers for pipeline %v: %v; retrying in %v", pipelineInfo.Pipeline.Name, err, d)
		return nil
	})
}

func (a *apiServer) deleteWorkersForPipeline(pipelineInfo *pps.PipelineInfo) error {
	rcName := ppsserver.PipelineRcName(pipelineInfo.Pipeline.Name, pipelineInfo.Version)
	if err := a.kubeClient.Services(a.namespace).Delete(rcName); err != nil {
		if !isNotFoundErr(err) {
			return err
		}
	}
	if pipelineInfo.Service != nil {
		if err := a.kubeClient.Services(a.namespace).Delete(rcName + "-user"); err != nil {
			if !isNotFoundErr(err) {
				return err
			}
		}
	}
	falseVal := false
	deleteOptions := &api.DeleteOptions{
		OrphanDependents: &falseVal,
	}
	if err := a.kubeClient.ReplicationControllers(a.namespace).Delete(rcName, deleteOptions); err != nil {
		if !isNotFoundErr(err) {
			return err
		}
	}
	return nil
}
<commit_msg>Fix bug in watches I introduced.<commit_after>package server

import (
	"context"
	"fmt"
	"path"
	"time"

	log "github.com/sirupsen/logrus"
	"k8s.io/kubernetes/pkg/api"
	labelspkg "k8s.io/kubernetes/pkg/labels"
	kube_watch "k8s.io/kubernetes/pkg/watch"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/backoff"
	col "github.com/pachyderm/pachyderm/src/server/pkg/collection"
	"github.com/pachyderm/pachyderm/src/server/pkg/dlock"
	"github.com/pachyderm/pachyderm/src/server/pkg/util"
	"github.com/pachyderm/pachyderm/src/server/pkg/watch"
	ppsserver "github.com/pachyderm/pachyderm/src/server/pps"
)

const (
	masterLockPath = "_master_lock"
)

var (
	failures = map[string]bool{
		"InvalidImageName": true,
		"ErrImagePull":     true,
	}
)

// The master process is responsible for creating/deleting workers as
// pipelines are created/removed.
func (a *apiServer) master() {
	masterLock := dlock.NewDLock(a.etcdClient, path.Join(a.etcdPrefix, masterLockPath))
	backoff.RetryNotify(func() error {
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()

		ctx, err := masterLock.Lock(ctx)
		if err != nil {
			return err
		}
		defer masterLock.Unlock(ctx)

		log.Infof("Launching PPS master process")

		pipelineWatcher, err := a.pipelines.ReadOnly(ctx).WatchWithPrev()
		if err != nil {
			return fmt.Errorf("error creating watch: %+v", err)
		}
		defer pipelineWatcher.Close()

		// watchChan will be nil if the Watch call below errors, this means
		// that we won't receive events from k8s and won't be able to detect
		// errors in pods. We could just return that error and retry but that
		// prevents pachyderm from creating pipelines when there's an issue
		// talking to k8s.
		var watchChan <-chan kube_watch.Event
		kubePipelineWatch, err := a.kubeClient.Pods(a.namespace).Watch(api.ListOptions{
			LabelSelector: labelspkg.SelectorFromSet(map[string]string{
				"component": "worker",
			}),
			Watch: true,
		})
		if err != nil {
			log.Errorf("failed to watch kuburnetes pods: %v", err)
		} else {
			watchChan = kubePipelineWatch.ResultChan()
			defer kubePipelineWatch.Stop()
		}

		for {
			select {
			case event := <-pipelineWatcher.Watch():
				if event.Err != nil {
					return fmt.Errorf("event err: %+v", event.Err)
				}
				switch event.Type {
				case watch.EventPut:
					var pipelineName string
					var pipelineInfo pps.PipelineInfo
					if err := event.Unmarshal(&pipelineName, &pipelineInfo); err != nil {
						return err
					}

					if pipelineInfo.Salt == "" {
						if _, err := col.NewSTM(ctx, a.etcdClient, func(stm col.STM) error {
							pipelines := a.pipelines.ReadWrite(stm)
							newPipelineInfo := new(pps.PipelineInfo)
							if err := pipelines.Get(pipelineInfo.Pipeline.Name, newPipelineInfo); err != nil {
								return fmt.Errorf("error getting pipeline %s: %+v", pipelineName, err)
							}
							if newPipelineInfo.Salt == "" {
								newPipelineInfo.Salt = uuid.NewWithoutDashes()
							}
							pipelines.Put(pipelineInfo.Pipeline.Name, newPipelineInfo)
							return nil
						}); err != nil {
							return err
						}
					}

					var prevPipelineInfo pps.PipelineInfo
					if event.PrevKey != nil {
						if err := event.UnmarshalPrev(&pipelineName, &prevPipelineInfo); err != nil {
							return err
						}
					}

					// If the pipeline has been stopped, delete workers
					if pipelineStateToStopped(pipelineInfo.State) {
						log.Infof("master: deleting workers for pipeline %s", pipelineInfo.Pipeline.Name)
						if err := a.deleteWorkersForPipeline(&pipelineInfo); err != nil {
							return err
						}
					}

					// If the pipeline has been restarted, create workers
					if !pipelineStateToStopped(pipelineInfo.State) && event.PrevKey != nil && pipelineStateToStopped(prevPipelineInfo.State) {
						if err := a.upsertWorkersForPipeline(&pipelineInfo); err != nil {
							if err := a.setPipelineFailure(ctx, pipelineInfo.Pipeline.Name, fmt.Sprintf("failed to create workers: %s", err.Error())); err != nil {
								return err
							}
							continue
						}
					}

					// If the pipeline has been updated, create new workers
					if pipelineInfo.Version > prevPipelineInfo.Version && !pipelineStateToStopped(pipelineInfo.State) {
						log.Infof("master: creating/updating workers for pipeline %s", pipelineInfo.Pipeline.Name)
						if event.PrevKey != nil {
							if err := a.deleteWorkersForPipeline(&prevPipelineInfo); err != nil {
								return err
							}
						}
						if err := a.upsertWorkersForPipeline(&pipelineInfo); err != nil {
							if err := a.setPipelineFailure(ctx, pipelineInfo.Pipeline.Name, fmt.Sprintf("failed to create workers: %s", err.Error())); err != nil {
								return err
							}
							continue
						}
					}
				case watch.EventDelete:
					var pipelineName string
					var pipelineInfo pps.PipelineInfo
					if err := event.UnmarshalPrev(&pipelineName, &pipelineInfo); err != nil {
						return err
					}
					if err := a.deleteWorkersForPipeline(&pipelineInfo); err != nil {
						return err
					}
				}
			case event := <-watchChan:
				// if we get an error we restart the watch, k8s watches seem to
				// sometimes get stuck in a loop returning events with Type =
				// "" we treat these as errors since otherwise we get an
				// endless stream of them and can't do anything.
				if event.Type == kube_watch.Error || event.Type == "" {
					kubePipelineWatch.Stop()
					kubePipelineWatch, err = a.kubeClient.Pods(a.namespace).Watch(api.ListOptions{
						LabelSelector: labelspkg.SelectorFromSet(map[string]string{
							"component": "worker",
						}),
						Watch: true,
					})
					if err != nil {
						log.Errorf("failed to watch kuburnetes pods: %v", err)
					} else {
						watchChan = kubePipelineWatch.ResultChan()
						defer kubePipelineWatch.Stop()
					}
				}
				pod, ok := event.Object.(*api.Pod)
				if !ok {
					continue
				}
				if pod.Status.Phase == api.PodFailed {
					log.Errorf("pod failed because: %s", pod.Status.Message)
				}
				for _, status := range pod.Status.ContainerStatuses {
					if status.Name == "user" && status.State.Waiting != nil && failures[status.State.Waiting.Reason] {
						if err := a.setPipelineFailure(ctx, pod.ObjectMeta.Annotations["pipelineName"], status.State.Waiting.Message); err != nil {
							return err
						}
					}
				}
			}
		}
	}, backoff.NewInfiniteBackOff(), func(err error, d time.Duration) error {
		log.Errorf("master: error running the master process: %v; retrying in %v", err, d)
		return nil
	})
}

func (a *apiServer) setPipelineFailure(ctx context.Context, pipelineName string, reason string) error {
	// Set pipeline state to failure
	_, err := col.NewSTM(ctx, a.etcdClient, func(stm col.STM) error {
		pipelines := a.pipelines.ReadWrite(stm)
		pipelineInfo := new(pps.PipelineInfo)
		if err := pipelines.Get(pipelineName, pipelineInfo); err != nil {
			return err
		}
		pipelineInfo.State = pps.PipelineState_PIPELINE_FAILURE
		pipelineInfo.Reason = reason
		pipelines.Put(pipelineName, pipelineInfo)
		return nil
	})
	return err
}

func (a *apiServer) upsertWorkersForPipeline(pipelineInfo *pps.PipelineInfo) error {
	var errCount int
	return backoff.RetryNotify(func() error {
		parallelism, err := ppsserver.GetExpectedNumWorkers(a.kubeClient, pipelineInfo.ParallelismSpec)
		if err != nil {
			return err
		}
		var resources *api.ResourceList
		if pipelineInfo.ResourceSpec != nil {
			resources, err = util.GetResourceListFromPipeline(pipelineInfo)
			if err != nil {
				return err
			}
		}

		// Retrieve the current state of the RC.  If the RC is scaled down,
		// we want to ensure that it remains scaled down.
		rc := a.kubeClient.ReplicationControllers(a.namespace)
		workerRc, err := rc.Get(ppsserver.PipelineRcName(pipelineInfo.Pipeline.Name, pipelineInfo.Version))
		if err == nil {
			if (workerRc.Spec.Template.Spec.Containers[0].Resources.Requests == nil) && workerRc.Spec.Replicas == 1 {
				parallelism = 1
				resources = nil
			}
		}

		options := a.getWorkerOptions(
			pipelineInfo.Pipeline.Name,
			ppsserver.PipelineRcName(pipelineInfo.Pipeline.Name, pipelineInfo.Version),
			int32(parallelism),
			resources,
			pipelineInfo.Transform,
			pipelineInfo.CacheSize,
			pipelineInfo.Service)
		// Set the pipeline name env
		options.workerEnv = append(options.workerEnv, api.EnvVar{
			Name:  client.PPSPipelineNameEnv,
			Value: pipelineInfo.Pipeline.Name,
		})
		return a.createWorkerRc(options)
	}, backoff.NewInfiniteBackOff(), func(err error, d time.Duration) error {
		errCount++
		if errCount >= 3 {
			return err
		}
		log.Errorf("error creating workers for pipeline %v: %v; retrying in %v", pipelineInfo.Pipeline.Name, err, d)
		return nil
	})
}

func (a *apiServer) deleteWorkersForPipeline(pipelineInfo *pps.PipelineInfo) error {
	rcName := ppsserver.PipelineRcName(pipelineInfo.Pipeline.Name, pipelineInfo.Version)
	if err := a.kubeClient.Services(a.namespace).Delete(rcName); err != nil {
		if !isNotFoundErr(err) {
			return err
		}
	}
	if pipelineInfo.Service != nil {
		if err := a.kubeClient.Services(a.namespace).Delete(rcName + "-user"); err != nil {
			if !isNotFoundErr(err) {
				return err
			}
		}
	}
	falseVal := false
	deleteOptions := &api.DeleteOptions{
		OrphanDependents: &falseVal,
	}
	if err := a.kubeClient.ReplicationControllers(a.namespace).Delete(rcName, deleteOptions); err != nil {
		if !isNotFoundErr(err) {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package suggestionbox

import (
	"bytes"
	"context"
	"encoding/json"
	"net/http"
	"net/url"
	"time"

	"github.com/pkg/errors"
)

// Model represents a single model inside Suggestionbox.
type Model struct {
	// ID is the ID of the model.
	ID string `json:"id,omitempty"`
	// Name is the human readable name of the Model.
	Name string `json:"name,omitempty"`
	// Options are optional Model settings to adjust the behaviour
	// of this Model within Suggestionbox.
	Options *ModelOptions `json:"options,omitempty"`
	// Choices are the options this Model will select from.
	Choices []Choice `json:"choices,omitempty"`
}

// Feature represents a single feature, to describe an input or a choice
// for example age:28 or location:"London".
type Feature struct {
	Key   string `json:"key,omitempty"`
	Value string `json:"value,omitempty"`
	Type  string `json:"type,omitempty"`
}

// Choice is an option with features.
type Choice struct {
	ID       string    `json:"id,omitempty"`
	Features []Feature `json:"features,omitempty"`
}

// ModelOptions describes the behaviours of a Model.
type ModelOptions struct {
	// Expiration is the time to wait for the reward before it expires.
	Expiration time.Duration `json:"expiration,omitempty"`

	// Epsilon enables proportionate exploiting vs exploring ratio.
	Epsilon float64 `json:"epsilon,omitempty"`

	// SoftmaxLambda enables adaptive exploiting vs exploring ratio.
	SoftmaxLambda float64 `json:"softmax_lambda,omitempty"`

	// Ngrams describes the n-grams for text analysis.
	Ngrams int `json:"ngrams,omitempty"`
	// Skipgrams describes the skip-grams for the text analysis.
	Skipgrams int `json:"skipgrams,omitempty"`
}

// CreateModel creates the Model in Suggestionbox.
// If no ID is set, one will be assigned in the return Model.
func (c *Client) CreateModel(ctx context.Context, model Model) (Model, error) {
	u, err := url.Parse(c.addr + "/suggestionbox/models")
	if err != nil {
		return model, err
	}
	if !u.IsAbs() {
		return model, errors.New("box address must be absolute")
	}
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(model); err != nil {
		return model, errors.Wrap(err, "encoding request body")
	}
	req, err := http.NewRequest(http.MethodPost, u.String(), &buf)
	if err != nil {
		return model, err
	}
	req = req.WithContext(ctx)
	req.Header.Set("Accept", "application/json; charset=utf-8")
	req.Header.Set("Content-Type", "application/json; charset=utf-8")
	resp, err := c.HTTPClient.Do(req)
	if err != nil {
		return model, err
	}
	defer resp.Body.Close()
	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		return model, errors.New(resp.Status)
	}
	var response struct {
		Success bool
		Error   string
		Model
	}
	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
		return model, errors.Wrap(err, "decoding response")
	}
	if !response.Success {
		return model, ErrSuggestionbox(response.Error)
	}
	return response.Model, nil
}
<commit_msg>added more docs<commit_after>package suggestionbox

import (
	"bytes"
	"context"
	"encoding/json"
	"net/http"
	"net/url"
	"time"

	"github.com/pkg/errors"
)

// Model represents a single model inside Suggestionbox.
type Model struct {
	// ID is the ID of the model.
	ID string `json:"id,omitempty"`
	// Name is the human readable name of the Model.
	Name string `json:"name,omitempty"`
	// Options are optional Model settings to adjust the behaviour
	// of this Model within Suggestionbox.
	Options *ModelOptions `json:"options,omitempty"`
	// Choices are the options this Model will select from.
	Choices []Choice `json:"choices,omitempty"`
}

// Feature represents a single feature, to describe an input or a choice
// for example age:28 or location:"London".
type Feature struct {
	// Key is the name of the Feature.
	Key string `json:"key,omitempty"`
	// Value is the string value of this Feature.
	Value string `json:"value,omitempty"`
	// Type is the type of the Feature.
	Type string `json:"type,omitempty"`
}

// Choice is an option with features.
type Choice struct {
	// ID is a unique ID for this choice.
	ID string `json:"id,omitempty"`
	// Features holds all the Feature objects that describe
	// this choice.
	Features []Feature `json:"features,omitempty"`
}

// ModelOptions describes the behaviours of a Model.
type ModelOptions struct {
	// Expiration is the time to wait for the reward before it expires.
	Expiration time.Duration `json:"expiration,omitempty"`

	// Epsilon enables proportionate exploiting vs exploring ratio.
	Epsilon float64 `json:"epsilon,omitempty"`

	// SoftmaxLambda enables adaptive exploiting vs exploring ratio.
	SoftmaxLambda float64 `json:"softmax_lambda,omitempty"`

	// Ngrams describes the n-grams for text analysis.
	Ngrams int `json:"ngrams,omitempty"`
	// Skipgrams describes the skip-grams for the text analysis.
	Skipgrams int `json:"skipgrams,omitempty"`
}

// CreateModel creates the Model in Suggestionbox.
// If no ID is set, one will be assigned in the return Model.
func (c *Client) CreateModel(ctx context.Context, model Model) (Model, error) {
	u, err := url.Parse(c.addr + "/suggestionbox/models")
	if err != nil {
		return model, err
	}
	if !u.IsAbs() {
		return model, errors.New("box address must be absolute")
	}
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(model); err != nil {
		return model, errors.Wrap(err, "encoding request body")
	}
	req, err := http.NewRequest(http.MethodPost, u.String(), &buf)
	if err != nil {
		return model, err
	}
	req = req.WithContext(ctx)
	req.Header.Set("Accept", "application/json; charset=utf-8")
	req.Header.Set("Content-Type", "application/json; charset=utf-8")
	resp, err := c.HTTPClient.Do(req)
	if err != nil {
		return model, err
	}
	defer resp.Body.Close()
	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		return model, errors.New(resp.Status)
	}
	var response struct {
		Success bool
		Error   string
		Model
	}
	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
		return model, errors.Wrap(err, "decoding response")
	}
	if !response.Success {
		return model, ErrSuggestionbox(response.Error)
	}
	return response.Model, nil
}
<|endoftext|>


<commit_before>package recommended

import (
	"net/http"
	"sort"

	"github.com/aerogo/aero"
	"github.com/animenotifier/arn"
	"github.com/animenotifier/notify.moe/components"
)

const maxRecommendations = 20

// Anime shows a list of recommended anime.
func Anime(ctx *aero.Context) string {
	nick := ctx.Get("nick")
	user, err := arn.GetUserByNick(nick)

	if err != nil {
		return ctx.Error(http.StatusUnauthorized, "Not logged in", err)
	}

	animeList := user.AnimeList()
	genreItems := animeList.Genres()
	genreAffinity := map[string]float64{}

	for genre, animeListItems := range genreItems {
		affinity := 0.0

		for _, item := range animeListItems {
			if item.Status == arn.AnimeListStatusDropped {
				affinity -= 5.0
				continue
			}

			if item.Rating.Overall != 0 {
				affinity += item.Rating.Overall
			} else {
				affinity += 5.0
			}
		}

		genreAffinity[genre] = affinity
	}

	// Get all anime
	recommendations := arn.AllAnime()

	// Affinity maps an anime ID to a number that indicates how likely a user is going to enjoy that anime.
	affinity := map[string]float64{}

	// Calculate affinity for each anime
	for _, anime := range recommendations {
		// Skip anime from my list (except planned anime)
		existing := animeList.Find(anime.ID)

		if existing != nil && existing.Status != arn.AnimeListStatusPlanned {
			continue
		}

		affinity[anime.ID] = float64(anime.Popularity.Total())

		// animeGenresAffinity := 0.0

		// if len(anime.Genres) > 0 {
		// 	for _, genre := range anime.Genres {
		// 		if genreAffinity[genre] > animeGenresAffinity {
		// 			animeGenresAffinity = genreAffinity[genre]
		// 		}
		// 	}

		// 	animeGenresAffinity = animeGenresAffinity / float64(len(anime.Genres))
		// }

		// affinity[anime.ID] = animeGenresAffinity
	}

	// Sort
	sort.Slice(recommendations, func(i, j int) bool {
		return affinity[recommendations[i].ID] > affinity[recommendations[j].ID]
	})

	// Take the top 10
	if len(recommendations) > maxRecommendations {
		recommendations = recommendations[:maxRecommendations]
	}

	return ctx.HTML(components.RecommendedAnime(recommendations, user))
}
<commit_msg>Improved recommendations<commit_after>package recommended

import (
	"fmt"
	"net/http"
	"sort"

	"github.com/aerogo/aero"
	"github.com/animenotifier/arn"
	"github.com/animenotifier/notify.moe/components"
)

const (
	maxRecommendations = 20
	worstGenreCount    = 5
)

// Anime shows a list of recommended anime.
func Anime(ctx *aero.Context) string {
	nick := ctx.Get("nick")
	user, err := arn.GetUserByNick(nick)

	if err != nil {
		return ctx.Error(http.StatusUnauthorized, "Not logged in", err)
	}

	animeList := user.AnimeList()
	genreItems := animeList.Genres()
	genreAffinity := map[string]float64{}
	worstGenres := []string{}

	for genre, animeListItems := range genreItems {
		affinity := 0.0

		for _, item := range animeListItems {
			// if item.Status == arn.AnimeListStatusDropped {
			// 	affinity -= 5.0
			// 	continue
			// }

			if item.Rating.Overall != 0 {
				affinity += item.Rating.Overall
			} else {
				affinity += 5.0
			}
		}

		genreAffinity[genre] = affinity
		worstGenres = append(worstGenres, genre)
	}

	sort.Slice(worstGenres, func(i, j int) bool {
		return genreAffinity[worstGenres[i]] < genreAffinity[worstGenres[j]]
	})

	if len(worstGenres) > worstGenreCount {
		worstGenres = worstGenres[:worstGenreCount]
	}

	fmt.Println(worstGenres)

	// Get all anime
	recommendations := arn.AllAnime()

	// Affinity maps an anime ID to a number that indicates how likely a user is going to enjoy that anime.
	affinity := map[string]float64{}

	// Calculate affinity for each anime
	for _, anime := range recommendations {
		// Skip anime that are upcoming
		if anime.Status == "upcoming" {
			continue
		}

		// Skip anime from my list (except planned anime)
		existing := animeList.Find(anime.ID)

		if existing != nil && existing.Status != arn.AnimeListStatusPlanned {
			continue
		}

		// Skip anime that don't have one of the top genres for that user
		worstGenreFound := false

		for _, genre := range anime.Genres {
			if arn.Contains(worstGenres, genre) {
				worstGenreFound = true
				break
			}
		}

		if worstGenreFound {
			continue
		}

		animeAffinity := 0.0

		// Planned anime go higher
		if existing != nil && existing.Status == arn.AnimeListStatusPlanned {
			animeAffinity += 75.0
		}

		animeAffinity += float64(anime.Popularity.Total())
		affinity[anime.ID] = animeAffinity
	}

	// Sort
	sort.Slice(recommendations, func(i, j int) bool {
		affinityA := affinity[recommendations[i].ID]
		affinityB := affinity[recommendations[j].ID]

		if affinityA == affinityB {
			return recommendations[i].Title.Canonical < recommendations[j].Title.Canonical
		}

		return affinityA > affinityB
	})

	// Take the top 10
	if len(recommendations) > maxRecommendations {
		recommendations = recommendations[:maxRecommendations]
	}

	return ctx.HTML(components.RecommendedAnime(recommendations, user))
}
<|endoftext|>


<commit_before>package swagger2

import (
	"encoding/json"
	"io/ioutil"
)

type Specification struct {
	Host     string          `json:"host,omitempty"`
	Info     Info            `json:"info,omitempty"`
	BasePath string          `json:"basePath,omitempty"`
	Schemes  []string        `json:"schemes,omitempty"`
	Paths    map[string]Path `json:"paths,omitempty"`
}

func NewSpecificationFromBytes(data []byte) (Specification, error) {
	spec := Specification{}
	err := json.Unmarshal(data, &spec)
	return spec, err
}

func ReadSwagger2Spec(filepath string) (Specification, error) {
	bytes, err := ioutil.ReadFile(filepath)
	if err != nil {
		return Specification{}, err
	}
	return NewSpecificationFromBytes(bytes)
}

type Info struct {
	Description    string `json:"description,omitempty"`
	Version        string `json:"version,omitempty"`
	Title          string `json:"title,omitempty"`
	TermsOfService string `json:"termsOfService,omitempty"`
}

type Path struct {
	Get    Endpoint `json:"get,omitempty"`
	Post   Endpoint `json:"post,omitempty"`
	Put    Endpoint `json:"put,omitempty"`
	Delete Endpoint `json:"delete,omitempty"`
}

type Endpoint struct {
	Tags        []string    `json:"tags,omitempty"`
	Summary     string      `json:"summary,omitempty"`
	OperationId string      `json:"operationId,omitempmty"`
	Description string      `json:"description,omitempty"`
	Consumes    []string    `json:"consumes,omitempty"`
	Produces    []string    `json:"produces,omitempty"`
	Parameters  []Parameter `json:"parameters"`
}

type Parameter struct {
	Name        string      `json:"name,omitempty"`
	Type        string      `json:"type,omitempty"`
	In          string      `json:"in,omitempty"`
	Description string      `json:"description,omitempty"`
	Required    bool        `json:"required,omitempty"`
	Default     interface{} `json:"default,omitempty"`
}
<commit_msg>swagger2: lint<commit_after>package swagger2

import (
	"encoding/json"
	"io/ioutil"
)

type Specification struct {
	Host     string          `json:"host,omitempty"`
	Info     Info            `json:"info,omitempty"`
	BasePath string          `json:"basePath,omitempty"`
	Schemes  []string        `json:"schemes,omitempty"`
	Paths    map[string]Path `json:"paths,omitempty"`
}

func NewSpecificationFromBytes(data []byte) (Specification, error) {
	spec := Specification{}
	err := json.Unmarshal(data, &spec)
	return spec, err
}

func ReadSwagger2Spec(filepath string) (Specification, error) {
	bytes, err := ioutil.ReadFile(filepath)
	if err != nil {
		return Specification{}, err
	}
	return NewSpecificationFromBytes(bytes)
}

type Info struct {
	Description    string `json:"description,omitempty"`
	Version        string `json:"version,omitempty"`
	Title          string `json:"title,omitempty"`
	TermsOfService string `json:"termsOfService,omitempty"`
}

type Path struct {
	Get    Endpoint `json:"get,omitempty"`
	Post   Endpoint `json:"post,omitempty"`
	Put    Endpoint `json:"put,omitempty"`
	Delete Endpoint `json:"delete,omitempty"`
}

type Endpoint struct {
	Tags        []string    `json:"tags,omitempty"`
	Summary     string      `json:"summary,omitempty"`
	OperationID string      `json:"operationId,omitempmty"`
	Description string      `json:"description,omitempty"`
	Consumes    []string    `json:"consumes,omitempty"`
	Produces    []string    `json:"produces,omitempty"`
	Parameters  []Parameter `json:"parameters"`
}

type Parameter struct {
	Name        string      `json:"name,omitempty"`
	Type        string      `json:"type,omitempty"`
	In          string      `json:"in,omitempty"`
	Description string      `json:"description,omitempty"`
	Required    bool        `json:"required,omitempty"`
	Default     interface{} `json:"default,omitempty"`
}
<|endoftext|>


<commit_before>package comm

import (
	"crypto/rand"
	"net"
	"testing"
	"time"

	log "github.com/schollz/logger"
	"github.com/stretchr/testify/assert"
)

func TestComm(t *testing.T) {
	token := make([]byte, 40000000)
	rand.Read(token)

	port := "8001"
	go func() {
		log.Debugf("starting TCP server on " + port)
		server, err := net.Listen("tcp", "0.0.0.0:"+port)
		if err != nil {
			log.Error(err)
		}
		defer server.Close()
		// spawn a new goroutine whenever a client connects
		for {
			connection, err := server.Accept()
			if err != nil {
				log.Error(err)
			}
			log.Debugf("client %s connected", connection.RemoteAddr().String())
			go func(port string, connection net.Conn) {
				c := New(connection)
				err = c.Send([]byte("hello, world"))
				assert.Nil(t, err)
				data, err := c.Receive()
				assert.Nil(t, err)
				assert.Equal(t, []byte("hello, computer"), data)
				data, err = c.Receive()
				assert.Nil(t, err)
				assert.Equal(t, []byte{'\x00'}, data)
				data, err = c.Receive()
				assert.Nil(t, err)
				assert.Equal(t, token, data)
			}(port, connection)
		}
	}()

	time.Sleep(300 * time.Millisecond)
	a, err := NewConnection("localhost:"+port, 10*time.Minute)
	assert.Nil(t, err)
	data, err := a.Receive()
	assert.Equal(t, []byte("hello, world"), data)
	assert.Nil(t, err)
	assert.Nil(t, a.Send([]byte("hello, computer")))
	assert.Nil(t, a.Send([]byte{'\x00'}))

	assert.Nil(t, a.Send(token))
	_ = a.Connection()
	a.Close()
	assert.NotNil(t, a.Send(token))
	_, err = a.Write(token)
	assert.NotNil(t, err)

}
<commit_msg>test should not exceed max bytes<commit_after>package comm

import (
	"crypto/rand"
	"net"
	"testing"
	"time"

	log "github.com/schollz/logger"
	"github.com/stretchr/testify/assert"
)

func TestComm(t *testing.T) {
	token := make([]byte, MAXBYTES)
	rand.Read(token)

	port := "8001"
	go func() {
		log.Debugf("starting TCP server on " + port)
		server, err := net.Listen("tcp", "0.0.0.0:"+port)
		if err != nil {
			log.Error(err)
		}
		defer server.Close()
		// spawn a new goroutine whenever a client connects
		for {
			connection, err := server.Accept()
			if err != nil {
				log.Error(err)
			}
			log.Debugf("client %s connected", connection.RemoteAddr().String())
			go func(port string, connection net.Conn) {
				c := New(connection)
				err = c.Send([]byte("hello, world"))
				assert.Nil(t, err)
				data, err := c.Receive()
				assert.Nil(t, err)
				assert.Equal(t, []byte("hello, computer"), data)
				data, err = c.Receive()
				assert.Nil(t, err)
				assert.Equal(t, []byte{'\x00'}, data)
				data, err = c.Receive()
				assert.Nil(t, err)
				assert.Equal(t, token, data)
			}(port, connection)
		}
	}()

	time.Sleep(300 * time.Millisecond)
	a, err := NewConnection("localhost:"+port, 10*time.Minute)
	assert.Nil(t, err)
	data, err := a.Receive()
	assert.Equal(t, []byte("hello, world"), data)
	assert.Nil(t, err)
	assert.Nil(t, a.Send([]byte("hello, computer")))
	assert.Nil(t, a.Send([]byte{'\x00'}))

	assert.Nil(t, a.Send(token))
	_ = a.Connection()
	a.Close()
	assert.NotNil(t, a.Send(token))
	_, err = a.Write(token)
	assert.NotNil(t, err)

}
<|endoftext|>


<commit_before>package conf

import (
	"errors"
	"os"
	"time"

	"github.com/joho/godotenv"
	"github.com/kelseyhightower/envconfig"
	"github.com/netlify/netlify-commons/nconf"
)

// OAuthProviderConfiguration holds all config related to external account providers.
type OAuthProviderConfiguration struct {
	ClientID    string `json:"client_id" split_words:"true"`
	Secret      string `json:"secret"`
	RedirectURI string `json:"redirect_uri" split_words:"true"`
	URL         string `json:"url"`
	Enabled     bool   `json:"enabled"`
}

// DBConfiguration holds all the database related configuration.
type DBConfiguration struct {
	Dialect     string `json:"dialect"`
	Driver      string `json:"driver" required:"true"`
	URL         string `json:"url" envconfig:"DATABASE_URL" required:"true"`
	Namespace   string `json:"namespace"`
	Automigrate bool   `json:"automigrate"`
}

// JWTConfiguration holds all the JWT related configuration.
type JWTConfiguration struct {
	Secret           string `json:"secret" required:"true"`
	Exp              int    `json:"exp"`
	Aud              string `json:"aud"`
	AdminGroupName   string `json:"admin_group_name" split_words:"true"`
	DefaultGroupName string `json:"default_group_name" split_words:"true"`
}

// GlobalConfiguration holds all the configuration that applies to all instances.
type GlobalConfiguration struct {
	API struct {
		Host     string
		Port     int `envconfig:"PORT" default:"8081"`
		Endpoint string
	}
	DB                DBConfiguration
	External          ExternalProviderConfiguration
	Logging           nconf.LoggingConfig `envconfig:"LOG"`
	OperatorToken     string              `split_words:"true" required:"true"`
	MultiInstanceMode bool
	SMTP              SMTPConfiguration
}

// EmailContentConfiguration holds the configuration for emails, both subjects and template URLs.
type EmailContentConfiguration struct {
	Invite       string `json:"invite"`
	Confirmation string `json:"confirmation"`
	Recovery     string `json:"recovery"`
	EmailChange  string `json:"email_change" split_words:"true"`
}

type ExternalProviderConfiguration struct {
	Bitbucket   OAuthProviderConfiguration `json:"bitbucket"`
	Github      OAuthProviderConfiguration `json:"github"`
	Gitlab      OAuthProviderConfiguration `json:"gitlab"`
	Google      OAuthProviderConfiguration `json:"google"`
	Facebook    OAuthProviderConfiguration `json:"facebook"`
	RedirectURL string                     `json:"redirect_url"`
}

type SMTPConfiguration struct {
	MaxFrequency time.Duration `json:"max_frequency" split_words:"true"`
	Host         string        `json:"host"`
	Port         int           `json:"port" default:"587"`
	User         string        `json:"user"`
	Pass         string        `json:"pass"`
	AdminEmail   string        `json:"admin_email" split_words:"true"`
}

// Configuration holds all the per-instance configuration.
type Configuration struct {
	SiteURL string            `json:"site_url" split_words:"true" required:"true"`
	JWT     JWTConfiguration  `json:"jwt"`
	SMTP    SMTPConfiguration `json:"smtp"`
	Mailer  struct {
		Autoconfirm bool                      `json:"autoconfirm"`
		Subjects    EmailContentConfiguration `json:"subjects"`
		Templates   EmailContentConfiguration `json:"templates"`
		URLPaths    EmailContentConfiguration `json:"url_paths"`
	} `json:"mailer"`
	External      ExternalProviderConfiguration `json:"external"`
	DisableSignup bool                          `json:"disable_signup" split_words:"true"`
	Webhook       WebhookConfig                 `json:"webhook" split_words:"true"`
	Cookie        struct {
		Key      string `json:"key"`
		Duration int    `json:"duration"`
	} `json:"cookies"`
}

func loadEnvironment(filename string) error {
	var err error
	if filename != "" {
		err = godotenv.Load(filename)
	} else {
		err = godotenv.Load()
		// handle if .env file does not exist, this is OK
		if os.IsNotExist(err) {
			return nil
		}
	}
	return err
}

type WebhookConfig struct {
	URL        string   `json:"url"`
	Retries    int      `json:"retries"`
	TimeoutSec int      `json:"timeout_sec"`
	Secret     string   `json:"jwt_secret"`
	Events     []string `json:"events"`
}

func (w *WebhookConfig) HasEvent(event string) bool {
	for _, name := range w.Events {
		if event == name {
			return true
		}
	}
	return false
}

// LoadGlobal loads configuration from file and environment variables.
func LoadGlobal(filename string) (*GlobalConfiguration, error) {
	if err := loadEnvironment(filename); err != nil {
		return nil, err
	}

	config := new(GlobalConfiguration)
	if err := envconfig.Process("gotrue", config); err != nil {
		return nil, err
	}

	if _, err := nconf.ConfigureLogging(&config.Logging); err != nil {
		return nil, err
	}

	if config.SMTP.MaxFrequency == 0 {
		config.SMTP.MaxFrequency = 15 * time.Minute
	}
	return config, nil
}

// LoadConfig loads per-instance configuration.
func LoadConfig(filename string) (*Configuration, error) {
	if err := loadEnvironment(filename); err != nil {
		return nil, err
	}

	config := new(Configuration)
	if err := envconfig.Process("gotrue", config); err != nil {
		return nil, err
	}
	config.ApplyDefaults()
	return config, nil
}

// ApplyDefaults sets defaults for a Configuration
func (config *Configuration) ApplyDefaults() {
	if config.JWT.AdminGroupName == "" {
		config.JWT.AdminGroupName = "admin"
	}

	if config.JWT.Exp == 0 {
		config.JWT.Exp = 3600
	}

	if config.Mailer.URLPaths.Invite == "" {
		config.Mailer.URLPaths.Invite = "/"
	}
	if config.Mailer.URLPaths.Confirmation == "" {
		config.Mailer.URLPaths.Confirmation = "/"
	}
	if config.Mailer.URLPaths.Recovery == "" {
		config.Mailer.URLPaths.Recovery = "/"
	}
	if config.Mailer.URLPaths.EmailChange == "" {
		config.Mailer.URLPaths.EmailChange = "/"
	}

	if config.SMTP.MaxFrequency == 0 {
		config.SMTP.MaxFrequency = 15 * time.Minute
	}

	if config.Cookie.Key == "" {
		config.Cookie.Key = "nf_jwt"
	}

	if config.Cookie.Duration == 0 {
		config.Cookie.Duration = 86400
	}
}

func (o *OAuthProviderConfiguration) Validate() error {
	if !o.Enabled {
		return errors.New("Provider is not enabled")
	}
	if o.ClientID == "" {
		return errors.New("Missing Oauth client ID")
	}
	if o.Secret == "" {
		return errors.New("Missing Oauth secret")
	}
	if o.RedirectURI == "" {
		return errors.New("Missing redirect URI")
	}
	return nil
}
<commit_msg>Rename jwt_secret to secret for consistency<commit_after>package conf

import (
	"errors"
	"os"
	"time"

	"github.com/joho/godotenv"
	"github.com/kelseyhightower/envconfig"
	"github.com/netlify/netlify-commons/nconf"
)

// OAuthProviderConfiguration holds all config related to external account providers.
type OAuthProviderConfiguration struct {
	ClientID    string `json:"client_id" split_words:"true"`
	Secret      string `json:"secret"`
	RedirectURI string `json:"redirect_uri" split_words:"true"`
	URL         string `json:"url"`
	Enabled     bool   `json:"enabled"`
}

// DBConfiguration holds all the database related configuration.
type DBConfiguration struct {
	Dialect     string `json:"dialect"`
	Driver      string `json:"driver" required:"true"`
	URL         string `json:"url" envconfig:"DATABASE_URL" required:"true"`
	Namespace   string `json:"namespace"`
	Automigrate bool   `json:"automigrate"`
}

// JWTConfiguration holds all the JWT related configuration.
type JWTConfiguration struct {
	Secret           string `json:"secret" required:"true"`
	Exp              int    `json:"exp"`
	Aud              string `json:"aud"`
	AdminGroupName   string `json:"admin_group_name" split_words:"true"`
	DefaultGroupName string `json:"default_group_name" split_words:"true"`
}

// GlobalConfiguration holds all the configuration that applies to all instances.
type GlobalConfiguration struct {
	API struct {
		Host     string
		Port     int `envconfig:"PORT" default:"8081"`
		Endpoint string
	}
	DB                DBConfiguration
	External          ExternalProviderConfiguration
	Logging           nconf.LoggingConfig `envconfig:"LOG"`
	OperatorToken     string              `split_words:"true" required:"true"`
	MultiInstanceMode bool
	SMTP              SMTPConfiguration
}

// EmailContentConfiguration holds the configuration for emails, both subjects and template URLs.
type EmailContentConfiguration struct {
	Invite       string `json:"invite"`
	Confirmation string `json:"confirmation"`
	Recovery     string `json:"recovery"`
	EmailChange  string `json:"email_change" split_words:"true"`
}

type ExternalProviderConfiguration struct {
	Bitbucket   OAuthProviderConfiguration `json:"bitbucket"`
	Github      OAuthProviderConfiguration `json:"github"`
	Gitlab      OAuthProviderConfiguration `json:"gitlab"`
	Google      OAuthProviderConfiguration `json:"google"`
	Facebook    OAuthProviderConfiguration `json:"facebook"`
	RedirectURL string                     `json:"redirect_url"`
}

type SMTPConfiguration struct {
	MaxFrequency time.Duration `json:"max_frequency" split_words:"true"`
	Host         string        `json:"host"`
	Port         int           `json:"port" default:"587"`
	User         string        `json:"user"`
	Pass         string        `json:"pass"`
	AdminEmail   string        `json:"admin_email" split_words:"true"`
}

// Configuration holds all the per-instance configuration.
type Configuration struct {
	SiteURL string            `json:"site_url" split_words:"true" required:"true"`
	JWT     JWTConfiguration  `json:"jwt"`
	SMTP    SMTPConfiguration `json:"smtp"`
	Mailer  struct {
		Autoconfirm bool                      `json:"autoconfirm"`
		Subjects    EmailContentConfiguration `json:"subjects"`
		Templates   EmailContentConfiguration `json:"templates"`
		URLPaths    EmailContentConfiguration `json:"url_paths"`
	} `json:"mailer"`
	External      ExternalProviderConfiguration `json:"external"`
	DisableSignup bool                          `json:"disable_signup" split_words:"true"`
	Webhook       WebhookConfig                 `json:"webhook" split_words:"true"`
	Cookie        struct {
		Key      string `json:"key"`
		Duration int    `json:"duration"`
	} `json:"cookies"`
}

func loadEnvironment(filename string) error {
	var err error
	if filename != "" {
		err = godotenv.Load(filename)
	} else {
		err = godotenv.Load()
		// handle if .env file does not exist, this is OK
		if os.IsNotExist(err) {
			return nil
		}
	}
	return err
}

type WebhookConfig struct {
	URL        string   `json:"url"`
	Retries    int      `json:"retries"`
	TimeoutSec int      `json:"timeout_sec"`
	Secret     string   `json:"secret"`
	Events     []string `json:"events"`
}

func (w *WebhookConfig) HasEvent(event string) bool {
	for _, name := range w.Events {
		if event == name {
			return true
		}
	}
	return false
}

// LoadGlobal loads configuration from file and environment variables.
func LoadGlobal(filename string) (*GlobalConfiguration, error) {
	if err := loadEnvironment(filename); err != nil {
		return nil, err
	}

	config := new(GlobalConfiguration)
	if err := envconfig.Process("gotrue", config); err != nil {
		return nil, err
	}

	if _, err := nconf.ConfigureLogging(&config.Logging); err != nil {
		return nil, err
	}

	if config.SMTP.MaxFrequency == 0 {
		config.SMTP.MaxFrequency = 15 * time.Minute
	}
	return config, nil
}

// LoadConfig loads per-instance configuration.
func LoadConfig(filename string) (*Configuration, error) {
	if err := loadEnvironment(filename); err != nil {
		return nil, err
	}

	config := new(Configuration)
	if err := envconfig.Process("gotrue", config); err != nil {
		return nil, err
	}
	config.ApplyDefaults()
	return config, nil
}

// ApplyDefaults sets defaults for a Configuration
func (config *Configuration) ApplyDefaults() {
	if config.JWT.AdminGroupName == "" {
		config.JWT.AdminGroupName = "admin"
	}

	if config.JWT.Exp == 0 {
		config.JWT.Exp = 3600
	}

	if config.Mailer.URLPaths.Invite == "" {
		config.Mailer.URLPaths.Invite = "/"
	}
	if config.Mailer.URLPaths.Confirmation == "" {
		config.Mailer.URLPaths.Confirmation = "/"
	}
	if config.Mailer.URLPaths.Recovery == "" {
		config.Mailer.URLPaths.Recovery = "/"
	}
	if config.Mailer.URLPaths.EmailChange == "" {
		config.Mailer.URLPaths.EmailChange = "/"
	}

	if config.SMTP.MaxFrequency == 0 {
		config.SMTP.MaxFrequency = 15 * time.Minute
	}

	if config.Cookie.Key == "" {
		config.Cookie.Key = "nf_jwt"
	}

	if config.Cookie.Duration == 0 {
		config.Cookie.Duration = 86400
	}
}

func (o *OAuthProviderConfiguration) Validate() error {
	if !o.Enabled {
		return errors.New("Provider is not enabled")
	}
	if o.ClientID == "" {
		return errors.New("Missing Oauth client ID")
	}
	if o.Secret == "" {
		return errors.New("Missing Oauth secret")
	}
	if o.RedirectURI == "" {
		return errors.New("Missing redirect URI")
	}
	return nil
}
<|endoftext|>


<commit_before>package conf

import (
	"os"
	"strconv"
	"time"

	"github.com/netlify/netlify-commons/nconf"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

// ExternalConfiguration holds all config related to external account providers.
type ExternalConfiguration struct {
	ClientID    string `json:"client_id"`
	Secret      string `json:"secret"`
	RedirectURI string `json:"redirect_uri"`
	URL         string `json:"url"`
}

// DBConfiguration holds all the database related configuration.
type DBConfiguration struct {
	Driver      string `json:"driver"`
	ConnURL     string `json:"url"`
	Namespace   string `json:"namespace"`
	Automigrate bool   `json:"automigrate"`
}

// JWTConfiguration holds all the JWT related configuration.
type JWTConfiguration struct {
	Secret             string `json:"secret"`
	Exp                int    `json:"exp"`
	Aud                string `json:"aud"`
	AdminGroupName     string `json:"admin_group_name"`
	AdminGroupDisabled bool   `json:"admin_group_disabled"`
	DefaultGroupName   string `json:"default_group_name"`
}

// GlobalConfiguration holds all the configuration that applies to all instances.
type GlobalConfiguration struct {
	API struct {
		Host     string `json:"host"`
		Port     int    `json:"port"`
		Endpoint string `json:"endpoint"`
	} `json:"api"`
	DB                DBConfiguration     `json:"db"`
	Logging           nconf.LoggingConfig `json:"log_conf"`
	NetlifySecret     string              `json:"netlify_secret"`
	MultiInstanceMode bool                `json:"-"`
}

// Configuration holds all the per-instance configuration.
type Configuration struct {
	SiteURL string           `json:"site_url"`
	JWT     JWTConfiguration `json:"jwt"`
	Mailer  struct {
		MaxFrequency time.Duration `json:"max_frequency"`
		Autoconfirm  bool          `json:"autoconfirm"`
		Host         string        `json:"host"`
		Port         int           `json:"port"`
		User         string        `json:"user"`
		Pass         string        `json:"pass"`
		MemberFolder string        `json:"member_folder"`
		AdminEmail   string        `json:"admin_email"`
		Subjects     struct {
			Invite       string `json:"invite"`
			Confirmation string `json:"confirmation"`
			Recovery     string `json:"recovery"`
			EmailChange  string `json:"email_change"`
		} `json:"subjects"`
		Templates struct {
			Invite       string `json:"invite"`
			Confirmation string `json:"confirmation"`
			Recovery     string `json:"recovery"`
			EmailChange  string `json:"email_change"`
		} `json:"templates"`
	} `json:"mailer"`
	External struct {
		Github    ExternalConfiguration `json:"github"`
		Bitbucket ExternalConfiguration `json:"bitbucket"`
		Gitlab    ExternalConfiguration `json:"gitlab"`
	} `json:"external"`
}

// LoadGlobalFromFile loads global configuration from the provided filename.
func LoadGlobalFromFile(name string) (*GlobalConfiguration, error) {
	cmd := &cobra.Command{}
	config := ""
	cmd.Flags().StringVar(&config, "config", "config.test.json", "Config file")
	cmd.Flags().Set("config", name)
	return LoadGlobal(cmd)
}

// LoadGlobal loads configuration from file and environment variables.
func LoadGlobal(cmd *cobra.Command) (*GlobalConfiguration, error) {
	config := new(GlobalConfiguration)

	if err := nconf.LoadConfig(cmd, "gotrue", config); err != nil {
		return nil, err
	}

	if config.DB.ConnURL == "" && os.Getenv("DATABASE_URL") != "" {
		config.DB.ConnURL = os.Getenv("DATABASE_URL")
	}

	if config.API.Port == 0 && os.Getenv("PORT") != "" {
		port, err := strconv.Atoi(os.Getenv("PORT"))
		if err != nil {
			return nil, errors.Wrap(err, "formatting PORT into int")
		}

		config.API.Port = port
	}

	if _, err := nconf.ConfigureLogging(&config.Logging); err != nil {
		return nil, err
	}

	return config, nil
}

// LoadConfigFromFile loads per-instance configuration from the provided filename.
func LoadConfigFromFile(name string) (*Configuration, error) {
	cmd := &cobra.Command{}
	config := ""
	cmd.Flags().StringVar(&config, "config", "config.test.json", "Config file")
	cmd.Flags().Set("config", name)
	return LoadConfig(cmd)
}

// LoadConfig loads per-instance configuration.
func LoadConfig(cmd *cobra.Command) (*Configuration, error) {
	config := new(Configuration)
	if err := nconf.LoadConfig(cmd, "gotrue", config); err != nil {
		return nil, err
	}

	config.ApplyDefaults()
	return config, nil
}

func (config *Configuration) ApplyDefaults() {
	if config.JWT.AdminGroupName == "" {
		config.JWT.AdminGroupName = "admin"
	}

	if config.JWT.Exp == 0 {
		config.JWT.Exp = 3600
	}

	if config.Mailer.MaxFrequency == 0 {
		config.Mailer.MaxFrequency = 15 * time.Minute
	}

	if config.Mailer.MemberFolder == "" {
		config.Mailer.MemberFolder = "/member"
	}

	if config.Mailer.Templates.Invite == "" {
		config.Mailer.Templates.Invite = "/.netlify/gotrue/templates/invite.html"
	}
	if config.Mailer.Templates.Confirmation == "" {
		config.Mailer.Templates.Confirmation = "/.netlify/gotrue/templates/confirm.html"
	}
	if config.Mailer.Templates.Recovery == "" {
		config.Mailer.Templates.Recovery = "/.netlify/gotrue/templates/recover.html"
	}
	if config.Mailer.Templates.EmailChange == "" {
		config.Mailer.Templates.EmailChange = "/.netlify/gotrue/templates/email-change.html"
	}
}
<commit_msg>Add default port of 8081<commit_after>package conf

import (
	"os"
	"strconv"
	"time"

	"github.com/netlify/netlify-commons/nconf"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

// ExternalConfiguration holds all config related to external account providers.
type ExternalConfiguration struct {
	ClientID    string `json:"client_id"`
	Secret      string `json:"secret"`
	RedirectURI string `json:"redirect_uri"`
	URL         string `json:"url"`
}

// DBConfiguration holds all the database related configuration.
type DBConfiguration struct {
	Driver      string `json:"driver"`
	ConnURL     string `json:"url"`
	Namespace   string `json:"namespace"`
	Automigrate bool   `json:"automigrate"`
}

// JWTConfiguration holds all the JWT related configuration.
type JWTConfiguration struct {
	Secret             string `json:"secret"`
	Exp                int    `json:"exp"`
	Aud                string `json:"aud"`
	AdminGroupName     string `json:"admin_group_name"`
	AdminGroupDisabled bool   `json:"admin_group_disabled"`
	DefaultGroupName   string `json:"default_group_name"`
}

// GlobalConfiguration holds all the configuration that applies to all instances.
type GlobalConfiguration struct {
	API struct {
		Host     string `json:"host"`
		Port     int    `json:"port"`
		Endpoint string `json:"endpoint"`
	} `json:"api"`
	DB                DBConfiguration     `json:"db"`
	Logging           nconf.LoggingConfig `json:"log_conf"`
	NetlifySecret     string              `json:"netlify_secret"`
	MultiInstanceMode bool                `json:"-"`
}

// Configuration holds all the per-instance configuration.
type Configuration struct {
	SiteURL string           `json:"site_url"`
	JWT     JWTConfiguration `json:"jwt"`
	Mailer  struct {
		MaxFrequency time.Duration `json:"max_frequency"`
		Autoconfirm  bool          `json:"autoconfirm"`
		Host         string        `json:"host"`
		Port         int           `json:"port"`
		User         string        `json:"user"`
		Pass         string        `json:"pass"`
		MemberFolder string        `json:"member_folder"`
		AdminEmail   string        `json:"admin_email"`
		Subjects     struct {
			Invite       string `json:"invite"`
			Confirmation string `json:"confirmation"`
			Recovery     string `json:"recovery"`
			EmailChange  string `json:"email_change"`
		} `json:"subjects"`
		Templates struct {
			Invite       string `json:"invite"`
			Confirmation string `json:"confirmation"`
			Recovery     string `json:"recovery"`
			EmailChange  string `json:"email_change"`
		} `json:"templates"`
	} `json:"mailer"`
	External struct {
		Github    ExternalConfiguration `json:"github"`
		Bitbucket ExternalConfiguration `json:"bitbucket"`
		Gitlab    ExternalConfiguration `json:"gitlab"`
	} `json:"external"`
}

// LoadGlobalFromFile loads global configuration from the provided filename.
func LoadGlobalFromFile(name string) (*GlobalConfiguration, error) {
	cmd := &cobra.Command{}
	config := ""
	cmd.Flags().StringVar(&config, "config", "config.test.json", "Config file")
	cmd.Flags().Set("config", name)
	return LoadGlobal(cmd)
}

// LoadGlobal loads configuration from file and environment variables.
func LoadGlobal(cmd *cobra.Command) (*GlobalConfiguration, error) {
	config := new(GlobalConfiguration)

	if err := nconf.LoadConfig(cmd, "gotrue", config); err != nil {
		return nil, err
	}

	if config.DB.ConnURL == "" && os.Getenv("DATABASE_URL") != "" {
		config.DB.ConnURL = os.Getenv("DATABASE_URL")
	}

	if config.API.Port == 0 && os.Getenv("PORT") != "" {
		port, err := strconv.Atoi(os.Getenv("PORT"))
		if err != nil {
			return nil, errors.Wrap(err, "formatting PORT into int")
		}

		config.API.Port = port
	}

	if config.API.Port == 0 && config.API.Host == "" {
		config.API.Port = 8081
	}

	if _, err := nconf.ConfigureLogging(&config.Logging); err != nil {
		return nil, err
	}

	return config, nil
}

// LoadConfigFromFile loads per-instance configuration from the provided filename.
func LoadConfigFromFile(name string) (*Configuration, error) {
	cmd := &cobra.Command{}
	config := ""
	cmd.Flags().StringVar(&config, "config", "config.test.json", "Config file")
	cmd.Flags().Set("config", name)
	return LoadConfig(cmd)
}

// LoadConfig loads per-instance configuration.
func LoadConfig(cmd *cobra.Command) (*Configuration, error) {
	config := new(Configuration)
	if err := nconf.LoadConfig(cmd, "gotrue", config); err != nil {
		return nil, err
	}

	config.ApplyDefaults()
	return config, nil
}

func (config *Configuration) ApplyDefaults() {
	if config.JWT.AdminGroupName == "" {
		config.JWT.AdminGroupName = "admin"
	}

	if config.JWT.Exp == 0 {
		config.JWT.Exp = 3600
	}

	if config.Mailer.MaxFrequency == 0 {
		config.Mailer.MaxFrequency = 15 * time.Minute
	}

	if config.Mailer.MemberFolder == "" {
		config.Mailer.MemberFolder = "/member"
	}

	if config.Mailer.Templates.Invite == "" {
		config.Mailer.Templates.Invite = "/.netlify/gotrue/templates/invite.html"
	}
	if config.Mailer.Templates.Confirmation == "" {
		config.Mailer.Templates.Confirmation = "/.netlify/gotrue/templates/confirm.html"
	}
	if config.Mailer.Templates.Recovery == "" {
		config.Mailer.Templates.Recovery = "/.netlify/gotrue/templates/recover.html"
	}
	if config.Mailer.Templates.EmailChange == "" {
		config.Mailer.Templates.EmailChange = "/.netlify/gotrue/templates/email-change.html"
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 The OpenPitrix Authors. All rights reserved.
// Use of this source code is governed by a Apache license
// that can be found in the LICENSE file.

package pilot

import (
	"context"
	"fmt"
	"time"

	"openpitrix.io/openpitrix/pkg/constants"
	"openpitrix.io/openpitrix/pkg/logger"
	"openpitrix.io/openpitrix/pkg/manager"
	"openpitrix.io/openpitrix/pkg/pb"
	"openpitrix.io/openpitrix/pkg/utils"
)

func NewPilotManagerClient(ctx context.Context) (pb.PilotManagerClient, error) {
	conn, err := manager.NewClient(ctx, constants.PilotManagerHost, constants.PilotManagerPort)
	if err != nil {
		return nil, err
	}
	return pb.NewPilotManagerClient(conn), err
}

func HandleSubtask(subtaskRequest *pb.HandleSubtaskRequest) error {
	ctx := context.Background()
	client, err := NewPilotManagerClient(ctx)
	if err != nil {
		return err
	}
	_, err = client.HandleSubtask(ctx, subtaskRequest)
	if err != nil {
		return err
	}
	return nil
}

func GetSubtaskStatus(subtaskStatusRequest *pb.GetSubtaskStatusRequest) (*pb.GetSubtaskStatusResponse, error) {
	ctx := context.Background()
	client, err := NewPilotManagerClient(ctx)
	if err != nil {
		return nil, err
	}
	subtaskStatusResponse, err := client.GetSubtaskStatus(ctx, subtaskStatusRequest)
	if err != nil {
		return nil, err
	}
	return subtaskStatusResponse, err
}

func WaitSubtask(taskId string, timeout time.Duration, waitInterval time.Duration) error {
	logger.Debug("Waiting for task [%s] finished", taskId)
	return utils.WaitForSpecificOrError(func() (bool, error) {
		taskStatusRequest := &pb.GetSubtaskStatusRequest{
			SubtaskId: []string{taskId},
		}
		taskStatusResponse, err := GetSubtaskStatus(taskStatusRequest)
		if err != nil {
			//network or api error, not considered task fail.
			return false, nil
		}
		if len(taskStatusResponse.SubtaskStatusSet) == 0 {
			return false, fmt.Errorf("Can not find task [%s]. ", taskId)
		}
		t := taskStatusResponse.SubtaskStatusSet[0]
		if t.Status == nil {
			logger.Errorf("Task [%s] status is nil", taskId)
			return false, nil
		}
		if t.Status.GetValue() == constants.StatusWorking || t.Status.GetValue() == constants.StatusPending {
			return false, nil
		}
		if t.Status.GetValue() == constants.StatusSuccessful {
			return true, nil
		}
		if t.Status.GetValue() == constants.StatusFailed {
			return false, fmt.Errorf("Task [%s] failed. ", taskId)
		}
		logger.Errorf("Unknown status [%s] for task [%s]. ", t.Status.GetValue(), taskId)
		return false, nil
	}, timeout, waitInterval)
}
<commit_msg>pkg/client/pilot: fix build<commit_after>// Copyright 2018 The OpenPitrix Authors. All rights reserved.
// Use of this source code is governed by a Apache license
// that can be found in the LICENSE file.

package pilot

import (
	"context"
	"fmt"
	"time"

	"openpitrix.io/openpitrix/pkg/constants"
	"openpitrix.io/openpitrix/pkg/logger"
	"openpitrix.io/openpitrix/pkg/manager"
	"openpitrix.io/openpitrix/pkg/pb"
	"openpitrix.io/openpitrix/pkg/utils"
)

func NewPilotManagerClient(ctx context.Context) (pb.PilotServiceClient, error) {
	conn, err := manager.NewClient(ctx, constants.PilotManagerHost, constants.PilotManagerPort)
	if err != nil {
		return nil, err
	}
	return pb.NewPilotServiceClient(conn), err
}

func HandleSubtask(subtaskRequest *pb.HandleSubtaskRequest) error {
	ctx := context.Background()
	client, err := NewPilotManagerClient(ctx)
	if err != nil {
		return err
	}
	_, err = client.HandleSubtask(ctx, subtaskRequest)
	if err != nil {
		return err
	}
	return nil
}

func GetSubtaskStatus(subtaskStatusRequest *pb.GetSubtaskStatusRequest) (*pb.GetSubtaskStatusResponse, error) {
	ctx := context.Background()
	client, err := NewPilotManagerClient(ctx)
	if err != nil {
		return nil, err
	}
	subtaskStatusResponse, err := client.GetSubtaskStatus(ctx, subtaskStatusRequest)
	if err != nil {
		return nil, err
	}
	return subtaskStatusResponse, err
}

func WaitSubtask(taskId string, timeout time.Duration, waitInterval time.Duration) error {
	logger.Debug("Waiting for task [%s] finished", taskId)
	return utils.WaitForSpecificOrError(func() (bool, error) {
		taskStatusRequest := &pb.GetSubtaskStatusRequest{
			SubtaskId: []string{taskId},
		}
		taskStatusResponse, err := GetSubtaskStatus(taskStatusRequest)
		if err != nil {
			//network or api error, not considered task fail.
			return false, nil
		}
		if len(taskStatusResponse.SubtaskStatusSet) == 0 {
			return false, fmt.Errorf("Can not find task [%s]. ", taskId)
		}
		t := taskStatusResponse.SubtaskStatusSet[0]
		if t.Status == nil {
			logger.Errorf("Task [%s] status is nil", taskId)
			return false, nil
		}
		if t.Status.GetValue() == constants.StatusWorking || t.Status.GetValue() == constants.StatusPending {
			return false, nil
		}
		if t.Status.GetValue() == constants.StatusSuccessful {
			return true, nil
		}
		if t.Status.GetValue() == constants.StatusFailed {
			return false, fmt.Errorf("Task [%s] failed. ", taskId)
		}
		logger.Errorf("Unknown status [%s] for task [%s]. ", t.Status.GetValue(), taskId)
		return false, nil
	}, timeout, waitInterval)
}
<|endoftext|>


<commit_before>package discovery

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"sort"
	"strings"
	"time"

	"net/http"

	log "github.com/Sirupsen/logrus"
	etcd "github.com/coreos/etcd/client"
	"github.com/matt-deboer/etcdcd/pkg/platform"
)

// Discovery provides correct startup details for etcd with respect to
// known vs. expected cluster membership
type Discovery struct {
	ConfigFile           string
	Platform             string
	ClientPort           int
	ServerPort           int
	ClientScheme         string
	ServerScheme         string
	MaxTries             int
	ProxyMode            bool
	MasterFilter         string
	DryRun               bool
	IgnoreNamingMismatch bool
	MinimumUptimeToJoin  time.Duration
}

func findMemberByName(members []etcd.Member, name string) *etcd.Member {
	for _, member := range members {
		if name == member.Name {
			return &member
		}
	}
	return nil
}

func containsMember(members []etcd.Member, member etcd.Member) bool {
	for _, m := range members {
		for _, peerURL := range m.PeerURLs {
			for _, memberPeerURL := range member.PeerURLs {
				if peerURL == memberPeerURL {
					return true
				}
			}
		}
	}
	return false
}

// DiscoverEnvironment produces an environment hash
func (d *Discovery) DiscoverEnvironment() (map[string]string, error) {

	p, err := platform.Get(d.Platform, d.ConfigFile)
	if p == nil {
		if err != nil {
			return nil, err
		}
		return nil, errors.New("No such platform: " + d.Platform)
	}

	var expectedMembers []etcd.Member
	for tries := 0; tries < d.MaxTries && len(expectedMembers) == 0; tries++ {
		if members, err := p.ExpectedMembers(d.MasterFilter, d.ClientScheme,
			d.ClientPort, d.ServerScheme, d.ServerPort); err == nil {
			for _, m := range members {
				// have to cast here because of golang type-system--ugh!
				expectedMembers = append(expectedMembers, etcd.Member(m))
				if len(m.PeerURLs) == 0 {
					log.Fatalf("Platform %s returned an invalid member which will be ignored: %#v", d.Platform, m)
				}
			}
		} else {
			return nil, err
		}
		if len(expectedMembers) == 0 {
			sleepTime := (2 * time.Second)
			if log.GetLevel() >= log.DebugLevel {
				log.Debugf("Failed to resolve expected members; sleeping for %s", sleepTime)
			}
			time.Sleep(sleepTime)
		}
	}

	if len(expectedMembers) == 0 {
		log.Fatal("Failed to determine expected members")
	} else if log.GetLevel() >= log.DebugLevel {
		log.Debugf("Expected cluster members: %v#", expectedMembers)
	}
	sort.Slice(expectedMembers, func(i, j int) bool { return expectedMembers[i].Name < expectedMembers[j].Name })

	localMaster := findMemberByName(expectedMembers, p.LocalInstanceName())
	membersAPI, currentMembers, uptime, err := d.resolveMembersAndAPI(expectedMembers, localMaster)

	environment := map[string]string{}
	environment["ETCD_NAME"] = p.LocalInstanceName()
	environment["ETCD_INITIAL_CLUSTER"] = initialClusterString(expectedMembers)

	if localMaster != nil {
		if log.GetLevel() >= log.DebugLevel {
			log.Debugf("Local master: %#v", *localMaster)
		}
		// this instance is an expected master
		if len(currentMembers) > 0 && !containsMember(currentMembers, *localMaster) && uptime >= d.MinimumUptimeToJoin {
			// there is an existing cluster
			if err = d.assertSaneClusterState(expectedMembers, currentMembers); err != nil {
				log.Fatal(err)
			}

			d.evictBadPeers(membersAPI, expectedMembers, currentMembers)
			log.Infof("Joining existing cluster as a master")
			// TODO: what if we encounter a state where not of the expected masters are
			// members of the current cluster?
			if err := d.joinExistingCluster(membersAPI, expectedMembers, localMaster); err != nil {
				log.Fatal(err)
			}
			environment["ETCD_INITIAL_CLUSTER_STATE"] = "existing"
		} else {
			log.Infof("Creating a new cluster")
			environment["ETCD_INITIAL_CLUSTER_STATE"] = "new"
		}
	} else if d.ProxyMode {
		log.Infof("Proxying existing cluster")
		environment["ETCD_INITIAL_CLUSTER_STATE"] = "existing"
		environment["ETCD_PROXY"] = "on"
	} else {
		return nil, fmt.Errorf(
			"Invalid cluster configuration: localhost (%s) is not an expected master, and not in proxy mode",
			p.LocalInstanceName())
	}
	return environment, nil
}

func initialClusterString(members []etcd.Member) string {
	if len(members) > 0 {
		initialCluster := make([]string, 0, len(members))
		for _, m := range members {
			member := fmt.Sprintf("%s=%s", m.Name, m.PeerURLs[0])
			initialCluster = append(initialCluster, member)
		}
		return strings.Join(initialCluster, ",")
	}
	return ""
}

// Check for mismatched names between expected and current members with
// matching peer URLs; also check for lack of intersection between
// expected and current members--indicating an invalid current (or expected)
// cluster state
func (d *Discovery) assertSaneClusterState(expectedMembers []etcd.Member, currentMembers []etcd.Member) error {
	partialMatchCount := 0
	for _, current := range currentMembers {
		for _, expected := range expectedMembers {
			matchingPeerURL := ""
			for _, expectedPeerURL := range expected.PeerURLs {
				for _, currentPeerURL := range current.PeerURLs {
					if expectedPeerURL == currentPeerURL {
						matchingPeerURL = expectedPeerURL
						partialMatchCount++
						break
					}
				}
				if len(matchingPeerURL) > 0 {
					break
				}
			}
			if len(matchingPeerURL) > 0 && len(current.Name) > 0 && current.Name != expected.Name {
				if !d.IgnoreNamingMismatch {
					return fmt.Errorf("Expected peer %s with peer URL %s already exists with a different name: %s",
						expected.Name, matchingPeerURL, current.Name)
				} else if log.GetLevel() >= log.DebugLevel {
					log.Debugf("Ignoring expected peer %s with peer URL %s already exists with a different name: %s",
						expected.Name, matchingPeerURL, current.Name)
				}
			}
		}
	}
	if partialMatchCount == 0 && len(expectedMembers) > 0 && len(currentMembers) > 0 {
		expectedJSON, _ := json.Marshal(expectedMembers)
		currentJSON, _ := json.Marshal(currentMembers)
		return fmt.Errorf("Invalid cluster state: found no intersection between peer URLs of expected members %s and current members %s",
			expectedJSON, currentJSON)
	}

	return nil
}

func (d *Discovery) evictBadPeers(membersAPI etcd.MembersAPI, expectedMembers []etcd.Member, currentMembers []etcd.Member) {

	for _, peer := range currentMembers {
		if !containsMember(expectedMembers, peer) {
			msg := fmt.Sprintf("Ejecting bad peer %s %v from the cluster:", peer.Name, peer.PeerURLs)
			if d.DryRun {
				log.Infof("DRY_RUN: would have ejected peer %s %v from the cluster", peer.Name, peer.PeerURLs)
			} else {
				for tries := 0; tries < d.MaxTries; tries++ {
					err := membersAPI.Remove(context.Background(), peer.ID)
					if err == nil {
						log.Infof("%s DONE", msg)
						break
					} else if (tries + 1) == d.MaxTries {
						log.Errorf("%s ERROR: %v", msg, err)
					}
				}
			}
		}
	}
}

func (d *Discovery) joinExistingCluster(membersAPI etcd.MembersAPI,
	expectedMembers []etcd.Member, localMember *etcd.Member) error {

	msg := "Joining existing cluster: "
	for tries := 0; tries < d.MaxTries; tries++ {
		if d.DryRun {
			log.Infof("DRY_RUN: would have added %s %v to the cluster", localMember.Name, localMember.PeerURLs)
		} else {
			_, err := membersAPI.Add(context.Background(), localMember.PeerURLs[0])
			if err == nil {
				log.Infof("%s DONE", msg)
				break
			} else if (tries + 1) == d.MaxTries {
				log.Errorf("%s ERROR: %v", msg, err)
				return err
			}
			if log.GetLevel() >= log.DebugLevel {
				log.Debugf("Retryable error attempting to add local master %#v: %v", localMember, err)
			}
			membersAPI, _, _, err = d.resolveMembersAndAPI(expectedMembers, localMember)
			if err != nil {
				log.Errorf("%s ERROR: %v", msg, err)
				return err
			}
		}
	}
	return nil
}

func (d *Discovery) resolveMembersAndAPI(expectedMembers []etcd.Member,
	localMember *etcd.Member) (membersAPI etcd.MembersAPI, currentMembers []etcd.Member, uptime time.Duration, err error) {

	ctx := context.Background()
	var lastErr error
	for tries := 0; tries <= d.MaxTries; tries++ {
		for _, member := range expectedMembers {
			// don't attempt self connection; afterall, this is intended as a pre-cursor
			// to the actual etcd service on the local host
			if localMember != nil && member.PeerURLs[0] != localMember.PeerURLs[0] {
				cfg := etcd.Config{
					Endpoints: member.ClientURLs,
					Transport: etcd.DefaultTransport,
					// set timeout per request to fail fast when the target endpoint is unavailable
					HeaderTimeoutPerRequest: time.Second,
				}
				etcdClient, err := etcd.New(cfg)
				if err != nil {
					if log.GetLevel() >= log.DebugLevel {
						log.Debugf("Error connecting to %s %v, %v", member.Name, member.ClientURLs, err)
					}
					lastErr = err
					continue
				}

				membersAPI = etcd.NewMembersAPI(etcdClient)
				leader, err := membersAPI.Leader(ctx)
				if err != nil {
					if log.GetLevel() >= log.DebugLevel {
						log.Debugf("Error getting leader %s %v, %v", member.Name, member.ClientURLs, err)
					}
					lastErr = err
					continue
				} else if leader == nil {
					if log.GetLevel() >= log.DebugLevel {
						log.Debugf("Error getting leader %s %v, %v", member.Name, member.ClientURLs, err)
					}
					lastErr = errors.New("Failed to resolve cluster leader")
				}

				currentMembers, err = membersAPI.List(ctx)
				if err != nil {
					if log.GetLevel() >= log.DebugLevel {
						log.Debugf("Error listing members %s %v, %v", member.Name, member.ClientURLs, err)
					}
					lastErr = err
					continue
				}

				uptime, err = getUptime(member.ClientURLs[0])
				if err != nil {
					if log.GetLevel() >= log.DebugLevel {
						log.Debugf("Error listing leader uptime %s %v, %v", member.Name, member.ClientURLs, err)
					}
					lastErr = err
					continue
				} else if log.GetLevel() >= log.DebugLevel {
					log.Debugf("Member %s reported current leader uptime is %s", member.Name, uptime)
				}

				// sanity-check the returned members; it may be partial in case of a yet-forming cluster
				hasInvalidMembers := false
				for _, m := range currentMembers {
					if len(m.Name) == 0 || len(m.PeerURLs) == 0 {
						if log.GetLevel() >= log.DebugLevel {
							log.Debugf("Returned actual member list contains invalid member: %#v", m)
						}
						hasInvalidMembers = true
						break
					}
				}
				if !hasInvalidMembers {
					if log.GetLevel() >= log.DebugLevel {
						log.Debugf("Actual cluster members: %#v", currentMembers)
					}
					return membersAPI, currentMembers, uptime, nil
				}
			}
		}
		if len(currentMembers) == 0 {
			// TODO: what's our timeout here?
			sleepTime := (1 * time.Second)
			if log.GetLevel() >= log.DebugLevel {
				log.Debugf("Failed to resolve members; sleeping for %s", sleepTime)
			}
			time.Sleep(sleepTime)
		} else {
			break
		}
	}
	return nil, nil, time.Millisecond, lastErr
}

func getUptime(endpoint string) (time.Duration, error) {
	resp, err := http.DefaultClient.Get(endpoint + "/v2/stats/self")
	if err != nil {
		return time.Millisecond, err
	}

	defer resp.Body.Close()
	contents, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return time.Millisecond, err
	}
	stats := make(map[string]interface{})
	err = json.Unmarshal(contents, &stats)
	if err != nil {
		return time.Millisecond, err
	}
	if leaderInfo, ok := stats["leaderInfo"]; ok {
		if uptimeString, ok := leaderInfo.(map[string]interface{})["uptime"]; ok {
			return time.ParseDuration(uptimeString.(string))
		}
	}
	return time.Millisecond, fmt.Errorf("Missing leader uptime info for endpiont %s", endpoint)
}
<commit_msg>lack of name does not an invalide member make<commit_after>package discovery

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"sort"
	"strings"
	"time"

	"net/http"

	log "github.com/Sirupsen/logrus"
	etcd "github.com/coreos/etcd/client"
	"github.com/matt-deboer/etcdcd/pkg/platform"
)

// Discovery provides correct startup details for etcd with respect to
// known vs. expected cluster membership
type Discovery struct {
	ConfigFile           string
	Platform             string
	ClientPort           int
	ServerPort           int
	ClientScheme         string
	ServerScheme         string
	MaxTries             int
	ProxyMode            bool
	MasterFilter         string
	DryRun               bool
	IgnoreNamingMismatch bool
	MinimumUptimeToJoin  time.Duration
}

func findMemberByName(members []etcd.Member, name string) *etcd.Member {
	for _, member := range members {
		if name == member.Name {
			return &member
		}
	}
	return nil
}

func containsMember(members []etcd.Member, member etcd.Member) bool {
	for _, m := range members {
		for _, peerURL := range m.PeerURLs {
			for _, memberPeerURL := range member.PeerURLs {
				if peerURL == memberPeerURL {
					return true
				}
			}
		}
	}
	return false
}

// DiscoverEnvironment produces an environment hash
func (d *Discovery) DiscoverEnvironment() (map[string]string, error) {

	p, err := platform.Get(d.Platform, d.ConfigFile)
	if p == nil {
		if err != nil {
			return nil, err
		}
		return nil, errors.New("No such platform: " + d.Platform)
	}

	var expectedMembers []etcd.Member
	for tries := 0; tries < d.MaxTries && len(expectedMembers) == 0; tries++ {
		if members, err := p.ExpectedMembers(d.MasterFilter, d.ClientScheme,
			d.ClientPort, d.ServerScheme, d.ServerPort); err == nil {
			for _, m := range members {
				// have to cast here because of golang type-system--ugh!
				expectedMembers = append(expectedMembers, etcd.Member(m))
				if len(m.PeerURLs) == 0 {
					log.Fatalf("Platform %s returned an invalid member which will be ignored: %#v", d.Platform, m)
				}
			}
		} else {
			return nil, err
		}
		if len(expectedMembers) == 0 {
			sleepTime := (2 * time.Second)
			if log.GetLevel() >= log.DebugLevel {
				log.Debugf("Failed to resolve expected members; sleeping for %s", sleepTime)
			}
			time.Sleep(sleepTime)
		}
	}

	if len(expectedMembers) == 0 {
		log.Fatal("Failed to determine expected members")
	} else if log.GetLevel() >= log.DebugLevel {
		log.Debugf("Expected cluster members: %v#", expectedMembers)
	}
	sort.Slice(expectedMembers, func(i, j int) bool { return expectedMembers[i].Name < expectedMembers[j].Name })

	localMaster := findMemberByName(expectedMembers, p.LocalInstanceName())
	membersAPI, currentMembers, uptime, err := d.resolveMembersAndAPI(expectedMembers, localMaster)

	environment := map[string]string{}
	environment["ETCD_NAME"] = p.LocalInstanceName()
	environment["ETCD_INITIAL_CLUSTER"] = initialClusterString(expectedMembers)

	if localMaster != nil {
		if log.GetLevel() >= log.DebugLevel {
			log.Debugf("Local master: %#v", *localMaster)
		}
		// this instance is an expected master
		if len(currentMembers) > 0 && !containsMember(currentMembers, *localMaster) && uptime >= d.MinimumUptimeToJoin {
			// there is an existing cluster
			if err = d.assertSaneClusterState(expectedMembers, currentMembers); err != nil {
				log.Fatal(err)
			}

			d.evictBadPeers(membersAPI, expectedMembers, currentMembers)
			log.Infof("Joining existing cluster as a master")
			// TODO: what if we encounter a state where not of the expected masters are
			// members of the current cluster?
			if err := d.joinExistingCluster(membersAPI, expectedMembers, localMaster); err != nil {
				log.Fatal(err)
			}
			environment["ETCD_INITIAL_CLUSTER_STATE"] = "existing"
		} else {
			log.Infof("Creating a new cluster")
			environment["ETCD_INITIAL_CLUSTER_STATE"] = "new"
		}
	} else if d.ProxyMode {
		log.Infof("Proxying existing cluster")
		environment["ETCD_INITIAL_CLUSTER_STATE"] = "existing"
		environment["ETCD_PROXY"] = "on"
	} else {
		return nil, fmt.Errorf(
			"Invalid cluster configuration: localhost (%s) is not an expected master, and not in proxy mode",
			p.LocalInstanceName())
	}
	return environment, nil
}

func initialClusterString(members []etcd.Member) string {
	if len(members) > 0 {
		initialCluster := make([]string, 0, len(members))
		for _, m := range members {
			member := fmt.Sprintf("%s=%s", m.Name, m.PeerURLs[0])
			initialCluster = append(initialCluster, member)
		}
		return strings.Join(initialCluster, ",")
	}
	return ""
}

// Check for mismatched names between expected and current members with
// matching peer URLs; also check for lack of intersection between
// expected and current members--indicating an invalid current (or expected)
// cluster state
func (d *Discovery) assertSaneClusterState(expectedMembers []etcd.Member, currentMembers []etcd.Member) error {
	partialMatchCount := 0
	for _, current := range currentMembers {
		for _, expected := range expectedMembers {
			matchingPeerURL := ""
			for _, expectedPeerURL := range expected.PeerURLs {
				for _, currentPeerURL := range current.PeerURLs {
					if expectedPeerURL == currentPeerURL {
						matchingPeerURL = expectedPeerURL
						partialMatchCount++
						break
					}
				}
				if len(matchingPeerURL) > 0 {
					break
				}
			}
			if len(matchingPeerURL) > 0 && len(current.Name) > 0 && current.Name != expected.Name {
				if !d.IgnoreNamingMismatch {
					return fmt.Errorf("Expected peer %s with peer URL %s already exists with a different name: %s",
						expected.Name, matchingPeerURL, current.Name)
				} else if log.GetLevel() >= log.DebugLevel {
					log.Debugf("Ignoring expected peer %s with peer URL %s already exists with a different name: %s",
						expected.Name, matchingPeerURL, current.Name)
				}
			}
		}
	}
	if partialMatchCount == 0 && len(expectedMembers) > 0 && len(currentMembers) > 0 {
		expectedJSON, _ := json.Marshal(expectedMembers)
		currentJSON, _ := json.Marshal(currentMembers)
		return fmt.Errorf("Invalid cluster state: found no intersection between peer URLs of expected members %s and current members %s",
			expectedJSON, currentJSON)
	}

	return nil
}

func (d *Discovery) evictBadPeers(membersAPI etcd.MembersAPI, expectedMembers []etcd.Member, currentMembers []etcd.Member) {

	for _, peer := range currentMembers {
		if !containsMember(expectedMembers, peer) {
			msg := fmt.Sprintf("Ejecting bad peer %s %v from the cluster:", peer.Name, peer.PeerURLs)
			if d.DryRun {
				log.Infof("DRY_RUN: would have ejected peer %s %v from the cluster", peer.Name, peer.PeerURLs)
			} else {
				for tries := 0; tries < d.MaxTries; tries++ {
					err := membersAPI.Remove(context.Background(), peer.ID)
					if err == nil {
						log.Infof("%s DONE", msg)
						break
					} else if (tries + 1) == d.MaxTries {
						log.Errorf("%s ERROR: %v", msg, err)
					}
				}
			}
		}
	}
}

func (d *Discovery) joinExistingCluster(membersAPI etcd.MembersAPI,
	expectedMembers []etcd.Member, localMember *etcd.Member) error {

	msg := "Joining existing cluster: "
	for tries := 0; tries < d.MaxTries; tries++ {
		if d.DryRun {
			log.Infof("DRY_RUN: would have added %s %v to the cluster", localMember.Name, localMember.PeerURLs)
		} else {
			_, err := membersAPI.Add(context.Background(), localMember.PeerURLs[0])
			if err == nil {
				log.Infof("%s DONE", msg)
				break
			} else if (tries + 1) == d.MaxTries {
				log.Errorf("%s ERROR: %v", msg, err)
				return err
			}
			if log.GetLevel() >= log.DebugLevel {
				log.Debugf("Retryable error attempting to add local master %#v: %v", localMember, err)
			}
			membersAPI, _, _, err = d.resolveMembersAndAPI(expectedMembers, localMember)
			if err != nil {
				log.Errorf("%s ERROR: %v", msg, err)
				return err
			}
		}
	}
	return nil
}

func (d *Discovery) resolveMembersAndAPI(expectedMembers []etcd.Member,
	localMember *etcd.Member) (membersAPI etcd.MembersAPI, currentMembers []etcd.Member, uptime time.Duration, err error) {

	ctx := context.Background()
	var lastErr error
	for tries := 0; tries <= d.MaxTries; tries++ {
		for _, member := range expectedMembers {
			// don't attempt self connection; afterall, this is intended as a pre-cursor
			// to the actual etcd service on the local host
			if localMember != nil && member.PeerURLs[0] != localMember.PeerURLs[0] {
				cfg := etcd.Config{
					Endpoints: member.ClientURLs,
					Transport: etcd.DefaultTransport,
					// set timeout per request to fail fast when the target endpoint is unavailable
					HeaderTimeoutPerRequest: time.Second,
				}
				etcdClient, err := etcd.New(cfg)
				if err != nil {
					if log.GetLevel() >= log.DebugLevel {
						log.Debugf("Error connecting to %s %v, %v", member.Name, member.ClientURLs, err)
					}
					lastErr = err
					continue
				}

				membersAPI = etcd.NewMembersAPI(etcdClient)
				leader, err := membersAPI.Leader(ctx)
				if err != nil {
					if log.GetLevel() >= log.DebugLevel {
						log.Debugf("Error getting leader %s %v, %v", member.Name, member.ClientURLs, err)
					}
					lastErr = err
					continue
				} else if leader == nil {
					if log.GetLevel() >= log.DebugLevel {
						log.Debugf("Error getting leader %s %v, %v", member.Name, member.ClientURLs, err)
					}
					lastErr = errors.New("Failed to resolve cluster leader")
				}

				currentMembers, err = membersAPI.List(ctx)
				if err != nil {
					if log.GetLevel() >= log.DebugLevel {
						log.Debugf("Error listing members %s %v, %v", member.Name, member.ClientURLs, err)
					}
					lastErr = err
					continue
				}

				uptime, err = getUptime(member.ClientURLs[0])
				if err != nil {
					if log.GetLevel() >= log.DebugLevel {
						log.Debugf("Error listing leader uptime %s %v, %v", member.Name, member.ClientURLs, err)
					}
					lastErr = err
					continue
				} else if log.GetLevel() >= log.DebugLevel {
					log.Debugf("Member %s reported current leader uptime is %s", member.Name, uptime)
				}

				if log.GetLevel() >= log.DebugLevel {
					log.Debugf("Actual cluster members: %#v", currentMembers)
				}
				return membersAPI, currentMembers, uptime, nil
			}
		}
		if len(currentMembers) == 0 {
			// TODO: what's our timeout here?
			sleepTime := (1 * time.Second)
			if log.GetLevel() >= log.DebugLevel {
				log.Debugf("Failed to resolve members; sleeping for %s", sleepTime)
			}
			time.Sleep(sleepTime)
		} else {
			break
		}
	}
	return nil, nil, time.Millisecond, lastErr
}

func getUptime(endpoint string) (time.Duration, error) {
	resp, err := http.DefaultClient.Get(endpoint + "/v2/stats/self")
	if err != nil {
		return time.Millisecond, err
	}

	defer resp.Body.Close()
	contents, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return time.Millisecond, err
	}
	stats := make(map[string]interface{})
	err = json.Unmarshal(contents, &stats)
	if err != nil {
		return time.Millisecond, err
	}
	if leaderInfo, ok := stats["leaderInfo"]; ok {
		if uptimeString, ok := leaderInfo.(map[string]interface{})["uptime"]; ok {
			return time.ParseDuration(uptimeString.(string))
		}
	}
	return time.Millisecond, fmt.Errorf("Missing leader uptime info for endpiont %s", endpoint)
}
<|endoftext|>


<commit_before>package localpeer

import (
	"sort"
	"sync"

	"nimona.io/internal/rand"
	"nimona.io/pkg/crypto"
	"nimona.io/pkg/object"
	"nimona.io/pkg/peer"
)

//go:generate mockgen -destination=../localpeermock/localpeermock_generated.go -package=localpeermock -source=localpeer.go
//go:generate genny -in=$GENERATORS/synclist/synclist.go -out=cids_generated.go -imp=nimona.io/pkg/object -pkg=localpeer gen "KeyType=object.CID"
//go:generate genny -in=$GENERATORS/synclist/synclist.go -out=certificates_generated.go -imp=nimona.io/pkg/peer -pkg=localpeer gen "KeyType=*object.Certificate"
//go:generate genny -in=$GENERATORS/synclist/synclist.go -out=addresses_generated.go -imp=nimona.io/pkg/peer -pkg=localpeer gen "KeyType=string"

type (
	LocalPeer interface {
		GetPrimaryPeerKey() crypto.PrivateKey
		PutPrimaryPeerKey(crypto.PrivateKey)
		GetPrimaryIdentityKey() crypto.PrivateKey
		PutPrimaryIdentityKey(crypto.PrivateKey)
		GetCertificates() []*object.Certificate
		PutCertificate(*object.Certificate)
		GetCIDs() []object.CID
		PutCIDs(...object.CID)
		GetContentTypes() []string
		PutContentTypes(...string)
		GetAddresses() []string
		PutAddresses(...string)
		GetRelays() []*peer.ConnectionInfo
		PutRelays(...*peer.ConnectionInfo)
		ConnectionInfo() *peer.ConnectionInfo
		ListenForUpdates() (<-chan UpdateEvent, func())
	}
	localPeer struct {
		keyLock            sync.RWMutex
		primaryPeerKey     crypto.PrivateKey
		primaryIdentityKey crypto.PrivateKey
		cids               *ObjectCIDSyncList
		contentTypes       *StringSyncList
		certificates       *ObjectCertificateSyncList
		addresses          *StringSyncList
		relays             []*peer.ConnectionInfo
		listeners          map[string]chan UpdateEvent
		listenersLock      sync.RWMutex
	}
	UpdateEvent string
)

const (
	EventContentTypesUpdated       UpdateEvent = "contentTypeUpdated"
	EventCIDsUpdated               UpdateEvent = "cidsUpdated"
	EventAddressesUpdated          UpdateEvent = "addressesUpdated"
	EventRelaysUpdated             UpdateEvent = "relaysUpdated"
	EventPrimaryIdentityKeyUpdated UpdateEvent = "primaryIdentityKeyUpdated"
)

func New() LocalPeer {
	return &localPeer{
		keyLock:       sync.RWMutex{},
		cids:          &ObjectCIDSyncList{},
		contentTypes:  &StringSyncList{},
		certificates:  &ObjectCertificateSyncList{},
		addresses:     &StringSyncList{},
		relays:        []*peer.ConnectionInfo{},
		listeners:     map[string]chan UpdateEvent{},
		listenersLock: sync.RWMutex{},
	}
}

func (s *localPeer) PutPrimaryPeerKey(k crypto.PrivateKey) {
	s.keyLock.Lock()
	s.primaryPeerKey = k
	s.keyLock.Unlock()
}

func (s *localPeer) PutPrimaryIdentityKey(k crypto.PrivateKey) {
	s.keyLock.Lock()
	s.primaryIdentityKey = k
	s.keyLock.Unlock()
	s.publishUpdate(EventPrimaryIdentityKeyUpdated)
}

func (s *localPeer) GetPrimaryPeerKey() crypto.PrivateKey {
	s.keyLock.RLock()
	defer s.keyLock.RUnlock() //nolint: gocritic
	return s.primaryPeerKey
}

func (s *localPeer) GetPrimaryIdentityKey() crypto.PrivateKey {
	s.keyLock.RLock()
	defer s.keyLock.RUnlock() //nolint: gocritic
	return s.primaryIdentityKey
}

func (s *localPeer) PutCertificate(c *object.Certificate) {
	s.certificates.Put(c)
}

func (s *localPeer) GetCertificates() []*object.Certificate {
	return s.certificates.List()
}

func (s *localPeer) GetAddresses() []string {
	as := s.addresses.List()
	sort.Strings(as)
	return as
}

func (s *localPeer) PutAddresses(addresses ...string) {
	for _, h := range addresses {
		s.addresses.Put(h)
	}
	s.publishUpdate(EventAddressesUpdated)
}

func (s *localPeer) GetCIDs() []object.CID {
	return s.cids.List()
}

func (s *localPeer) PutCIDs(cids ...object.CID) {
	for _, h := range cids {
		s.cids.Put(h)
	}
	s.publishUpdate(EventCIDsUpdated)
}

func (s *localPeer) GetContentTypes() []string {
	return s.contentTypes.List()
}

func (s *localPeer) PutContentTypes(contentTypes ...string) {
	for _, h := range contentTypes {
		s.contentTypes.Put(h)
	}
	s.publishUpdate(EventContentTypesUpdated)
}

func (s *localPeer) GetRelays() []*peer.ConnectionInfo {
	s.keyLock.RLock()
	defer s.keyLock.RUnlock()
	return s.relays
}

func (s *localPeer) PutRelays(relays ...*peer.ConnectionInfo) {
	s.keyLock.Lock()
	defer s.keyLock.Unlock()
	s.relays = append(s.relays, relays...)
	s.publishUpdate(EventRelaysUpdated)
}

func (s *localPeer) ConnectionInfo() *peer.ConnectionInfo {
	return &peer.ConnectionInfo{
		PublicKey: s.GetPrimaryPeerKey().PublicKey(),
		Addresses: s.GetAddresses(),
		Relays:    s.GetRelays(),
		ObjectFormats: []string{
			"json",
		},
	}
}

func (s *localPeer) publishUpdate(e UpdateEvent) {
	s.listenersLock.RLock()
	defer s.listenersLock.RUnlock()
	for _, l := range s.listeners {
		select {
		case l <- e:
		default:
		}
	}
}

func (s *localPeer) ListenForUpdates() (
	updates <-chan UpdateEvent,
	cancel func(),
) {
	c := make(chan UpdateEvent)
	s.listenersLock.Lock()
	defer s.listenersLock.Unlock()
	id := rand.String(8)
	s.listeners[id] = c
	f := func() {
		s.listenersLock.Lock()
		defer s.listenersLock.Unlock()
		delete(s.listeners, id)
	}
	return c, f
}
<commit_msg>chore(localpeer): add TODO for merging methods<commit_after>package localpeer

import (
	"sort"
	"sync"

	"nimona.io/internal/rand"
	"nimona.io/pkg/crypto"
	"nimona.io/pkg/object"
	"nimona.io/pkg/peer"
)

//go:generate mockgen -destination=../localpeermock/localpeermock_generated.go -package=localpeermock -source=localpeer.go
//go:generate genny -in=$GENERATORS/synclist/synclist.go -out=cids_generated.go -imp=nimona.io/pkg/object -pkg=localpeer gen "KeyType=object.CID"
//go:generate genny -in=$GENERATORS/synclist/synclist.go -out=certificates_generated.go -imp=nimona.io/pkg/peer -pkg=localpeer gen "KeyType=*object.Certificate"
//go:generate genny -in=$GENERATORS/synclist/synclist.go -out=addresses_generated.go -imp=nimona.io/pkg/peer -pkg=localpeer gen "KeyType=string"

type (
	LocalPeer interface {
		// TODO merge peer/id methods, use .Usage to distinguish
		GetPrimaryPeerKey() crypto.PrivateKey
		PutPrimaryPeerKey(crypto.PrivateKey)
		GetPrimaryIdentityKey() crypto.PrivateKey
		PutPrimaryIdentityKey(crypto.PrivateKey)
		GetCertificates() []*object.Certificate
		PutCertificate(*object.Certificate)
		GetCIDs() []object.CID
		PutCIDs(...object.CID)
		GetContentTypes() []string
		PutContentTypes(...string)
		GetAddresses() []string
		PutAddresses(...string)
		GetRelays() []*peer.ConnectionInfo
		PutRelays(...*peer.ConnectionInfo)
		ConnectionInfo() *peer.ConnectionInfo
		ListenForUpdates() (<-chan UpdateEvent, func())
	}
	localPeer struct {
		keyLock            sync.RWMutex
		primaryPeerKey     crypto.PrivateKey
		primaryIdentityKey crypto.PrivateKey
		cids               *ObjectCIDSyncList
		contentTypes       *StringSyncList
		certificates       *ObjectCertificateSyncList
		addresses          *StringSyncList
		relays             []*peer.ConnectionInfo
		listeners          map[string]chan UpdateEvent
		listenersLock      sync.RWMutex
	}
	UpdateEvent string
)

const (
	EventContentTypesUpdated       UpdateEvent = "contentTypeUpdated"
	EventCIDsUpdated               UpdateEvent = "cidsUpdated"
	EventAddressesUpdated          UpdateEvent = "addressesUpdated"
	EventRelaysUpdated             UpdateEvent = "relaysUpdated"
	EventPrimaryIdentityKeyUpdated UpdateEvent = "primaryIdentityKeyUpdated"
)

func New() LocalPeer {
	return &localPeer{
		keyLock:       sync.RWMutex{},
		cids:          &ObjectCIDSyncList{},
		contentTypes:  &StringSyncList{},
		certificates:  &ObjectCertificateSyncList{},
		addresses:     &StringSyncList{},
		relays:        []*peer.ConnectionInfo{},
		listeners:     map[string]chan UpdateEvent{},
		listenersLock: sync.RWMutex{},
	}
}

func (s *localPeer) PutPrimaryPeerKey(k crypto.PrivateKey) {
	s.keyLock.Lock()
	s.primaryPeerKey = k
	s.keyLock.Unlock()
}

func (s *localPeer) PutPrimaryIdentityKey(k crypto.PrivateKey) {
	s.keyLock.Lock()
	s.primaryIdentityKey = k
	s.keyLock.Unlock()
	s.publishUpdate(EventPrimaryIdentityKeyUpdated)
}

func (s *localPeer) GetPrimaryPeerKey() crypto.PrivateKey {
	s.keyLock.RLock()
	defer s.keyLock.RUnlock() //nolint: gocritic
	return s.primaryPeerKey
}

func (s *localPeer) GetPrimaryIdentityKey() crypto.PrivateKey {
	s.keyLock.RLock()
	defer s.keyLock.RUnlock() //nolint: gocritic
	return s.primaryIdentityKey
}

func (s *localPeer) PutCertificate(c *object.Certificate) {
	s.certificates.Put(c)
}

func (s *localPeer) GetCertificates() []*object.Certificate {
	return s.certificates.List()
}

func (s *localPeer) GetAddresses() []string {
	as := s.addresses.List()
	sort.Strings(as)
	return as
}

func (s *localPeer) PutAddresses(addresses ...string) {
	for _, h := range addresses {
		s.addresses.Put(h)
	}
	s.publishUpdate(EventAddressesUpdated)
}

func (s *localPeer) GetCIDs() []object.CID {
	return s.cids.List()
}

func (s *localPeer) PutCIDs(cids ...object.CID) {
	for _, h := range cids {
		s.cids.Put(h)
	}
	s.publishUpdate(EventCIDsUpdated)
}

func (s *localPeer) GetContentTypes() []string {
	return s.contentTypes.List()
}

func (s *localPeer) PutContentTypes(contentTypes ...string) {
	for _, h := range contentTypes {
		s.contentTypes.Put(h)
	}
	s.publishUpdate(EventContentTypesUpdated)
}

func (s *localPeer) GetRelays() []*peer.ConnectionInfo {
	s.keyLock.RLock()
	defer s.keyLock.RUnlock()
	return s.relays
}

func (s *localPeer) PutRelays(relays ...*peer.ConnectionInfo) {
	s.keyLock.Lock()
	defer s.keyLock.Unlock()
	s.relays = append(s.relays, relays...)
	s.publishUpdate(EventRelaysUpdated)
}

func (s *localPeer) ConnectionInfo() *peer.ConnectionInfo {
	return &peer.ConnectionInfo{
		PublicKey: s.GetPrimaryPeerKey().PublicKey(),
		Addresses: s.GetAddresses(),
		Relays:    s.GetRelays(),
		ObjectFormats: []string{
			"json",
		},
	}
}

func (s *localPeer) publishUpdate(e UpdateEvent) {
	s.listenersLock.RLock()
	defer s.listenersLock.RUnlock()
	for _, l := range s.listeners {
		select {
		case l <- e:
		default:
		}
	}
}

func (s *localPeer) ListenForUpdates() (
	updates <-chan UpdateEvent,
	cancel func(),
) {
	c := make(chan UpdateEvent)
	s.listenersLock.Lock()
	defer s.listenersLock.Unlock()
	id := rand.String(8)
	s.listeners[id] = c
	f := func() {
		s.listenersLock.Lock()
		defer s.listenersLock.Unlock()
		delete(s.listeners, id)
	}
	return c, f
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package resources contains various utilities for dealing with Kubernetes resources.
package resources
<commit_msg>Remove a superfluous file (#6999)<commit_after><|endoftext|>