<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package policy

import (
	"fmt"
	"io/ioutil"

	"k8s.io/apimachinery/pkg/runtime/schema"
	auditinternal "k8s.io/apiserver/pkg/apis/audit"
	auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
	auditv1alpha1 "k8s.io/apiserver/pkg/apis/audit/v1alpha1"
	auditv1beta1 "k8s.io/apiserver/pkg/apis/audit/v1beta1"
	"k8s.io/apiserver/pkg/apis/audit/validation"
	"k8s.io/apiserver/pkg/audit"

	"k8s.io/klog/v2"
)

var (
	apiGroupVersions = []schema.GroupVersion{
		auditv1beta1.SchemeGroupVersion,
		auditv1alpha1.SchemeGroupVersion,
		auditv1.SchemeGroupVersion,
	}
	apiGroupVersionSet = map[schema.GroupVersion]bool{}
)

func init() {
	for _, gv := range apiGroupVersions {
		apiGroupVersionSet[gv] = true
	}
}

func LoadPolicyFromFile(filePath string) (*auditinternal.Policy, error) {
	if filePath == "" {
		return nil, fmt.Errorf("file path not specified")
	}
	policyDef, err := ioutil.ReadFile(filePath)
	if err != nil {
		return nil, fmt.Errorf("failed to read file path %q: %+v", filePath, err)
	}

	ret, err := LoadPolicyFromBytes(policyDef)
	if err != nil {
		return nil, fmt.Errorf("%v: from file %v", err.Error(), filePath)
	}

	return ret, nil
}

func LoadPolicyFromBytes(policyDef []byte) (*auditinternal.Policy, error) {
	policy := &auditinternal.Policy{}
	decoder := audit.Codecs.UniversalDecoder(apiGroupVersions...)

	_, gvk, err := decoder.Decode(policyDef, nil, policy)
	if err != nil {
		return nil, fmt.Errorf("failed decoding: %v", err)
	}

	// Ensure the policy file contained an apiVersion and kind.
	if !apiGroupVersionSet[schema.GroupVersion{Group: gvk.Group, Version: gvk.Version}] {
		return nil, fmt.Errorf("unknown group version field %v in policy", gvk)
	}

	if err := validation.ValidatePolicy(policy); err != nil {
		return nil, err.ToAggregate()
	}

	policyCnt := len(policy.Rules)
	if policyCnt == 0 {
		return nil, fmt.Errorf("loaded illegal policy with 0 rules")
	}
	klog.V(4).Infof("Loaded %d audit policy rules", policyCnt)
	return policy, nil
}
<commit_msg>staging/src/k8s.io/apiserver/pkg/audit/policy/reader.go migrate logs to structured logging<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package policy

import (
	"fmt"
	"io/ioutil"

	"k8s.io/apimachinery/pkg/runtime/schema"
	auditinternal "k8s.io/apiserver/pkg/apis/audit"
	auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
	auditv1alpha1 "k8s.io/apiserver/pkg/apis/audit/v1alpha1"
	auditv1beta1 "k8s.io/apiserver/pkg/apis/audit/v1beta1"
	"k8s.io/apiserver/pkg/apis/audit/validation"
	"k8s.io/apiserver/pkg/audit"

	"k8s.io/klog/v2"
)

var (
	apiGroupVersions = []schema.GroupVersion{
		auditv1beta1.SchemeGroupVersion,
		auditv1alpha1.SchemeGroupVersion,
		auditv1.SchemeGroupVersion,
	}
	apiGroupVersionSet = map[schema.GroupVersion]bool{}
)

func init() {
	for _, gv := range apiGroupVersions {
		apiGroupVersionSet[gv] = true
	}
}

func LoadPolicyFromFile(filePath string) (*auditinternal.Policy, error) {
	if filePath == "" {
		return nil, fmt.Errorf("file path not specified")
	}
	policyDef, err := ioutil.ReadFile(filePath)
	if err != nil {
		return nil, fmt.Errorf("failed to read file path %q: %+v", filePath, err)
	}

	ret, err := LoadPolicyFromBytes(policyDef)
	if err != nil {
		return nil, fmt.Errorf("%v: from file %v", err.Error(), filePath)
	}

	return ret, nil
}

func LoadPolicyFromBytes(policyDef []byte) (*auditinternal.Policy, error) {
	policy := &auditinternal.Policy{}
	decoder := audit.Codecs.UniversalDecoder(apiGroupVersions...)

	_, gvk, err := decoder.Decode(policyDef, nil, policy)
	if err != nil {
		return nil, fmt.Errorf("failed decoding: %v", err)
	}

	// Ensure the policy file contained an apiVersion and kind.
	if !apiGroupVersionSet[schema.GroupVersion{Group: gvk.Group, Version: gvk.Version}] {
		return nil, fmt.Errorf("unknown group version field %v in policy", gvk)
	}

	if err := validation.ValidatePolicy(policy); err != nil {
		return nil, err.ToAggregate()
	}

	policyCnt := len(policy.Rules)
	if policyCnt == 0 {
		return nil, fmt.Errorf("loaded illegal policy with 0 rules")
	}

	klog.V(4).InfoS("Load audit policy rules success", "policyCnt", policyCnt)
	return policy, nil
}
<|endoftext|>


<commit_before>package controller

import (
	"fmt"
	"time"

	log "github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	metav1unstructured "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/client-go/dynamic"
	restclientset "k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"

	tfv1alpha2 "github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1alpha2"
	tfjobinformers "github.com/kubeflow/tf-operator/pkg/client/informers/externalversions"
	tfjobinformersv1alpha2 "github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/kubeflow/v1alpha2"
	"github.com/kubeflow/tf-operator/pkg/util/unstructured"
)

const (
	resyncPeriod = 30 * time.Second
)

var (
	errGetFromKey    = fmt.Errorf("Failed to get TFJob from key")
	errNotExists     = fmt.Errorf("The object is not found")
	errFailedMarshal = fmt.Errorf("Failed to marshal the object to TFJob")
)

func NewUnstructuredTFJobInformer(restConfig *restclientset.Config) tfjobinformersv1alpha2.TFJobInformer {
	dynClientPool := dynamic.NewDynamicClientPool(restConfig)
	dclient, err := dynClientPool.ClientForGroupVersionKind(controllerKind)
	if err != nil {
		panic(err)
	}
	resource := &metav1.APIResource{
		Name:         tfv1alpha2.Plural,
		SingularName: tfv1alpha2.Singular,
		Namespaced:   true,
		Group:        tfv1alpha2.GroupName,
		Version:      tfv1alpha2.GroupVersion,
	}
	informer := unstructured.NewTFJobInformer(
		resource,
		dclient,
		metav1.NamespaceAll,
		resyncPeriod,
		cache.Indexers{},
	)
	return informer
}

// NewTFJobInformer returns TFJobInformer from the given factory.
func (tc *TFJobController) NewTFJobInformer(tfJobInformerFactory tfjobinformers.SharedInformerFactory) tfjobinformersv1alpha2.TFJobInformer {
	return tfJobInformerFactory.Kubeflow().V1alpha2().TFJobs()
}

func (tc *TFJobController) getTFJobFromName(namespace, name string) (*tfv1alpha2.TFJob, error) {
	key := fmt.Sprintf("%s/%s", namespace, name)
	return tc.getTFJobFromKey(key)
}

func (tc *TFJobController) getTFJobFromKey(key string) (*tfv1alpha2.TFJob, error) {
	// Check if the key exists.
	obj, exists, err := tc.tfJobInformer.GetIndexer().GetByKey(key)
	if err != nil {
		log.Errorf("Failed to get TFJob '%s' from informer index: %+v", key, err)
		return nil, errGetFromKey
	}
	if !exists {
		// This happens after a tfjob was deleted, but the work queue still had an entry for it.
		return nil, errNotExists
	}

	tfjob, err := tfJobFromUnstructured(obj)
	if err != nil {
		return nil, err
	}
	return tfjob, nil
}

func tfJobFromUnstructured(obj interface{}) (*tfv1alpha2.TFJob, error) {
	// Check if the spec is valid.
	un, ok := obj.(*metav1unstructured.Unstructured)
	if !ok {
		log.Warn("The object in index is not an unstructured")
		return nil, errGetFromKey
	}
	var tfjob tfv1alpha2.TFJob
	err := runtime.DefaultUnstructuredConverter.FromUnstructured(un.Object, &tfjob)
	if err != nil || tfjob.Spec.TFReplicaSpecs == nil {
		return &tfjob, errFailedMarshal
	}
	return &tfjob, nil
}
<commit_msg>informer: Add comments and TODO (#684)<commit_after>package controller

import (
	"fmt"
	"time"

	log "github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	metav1unstructured "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/client-go/dynamic"
	restclientset "k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"

	tfv1alpha2 "github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1alpha2"
	tfjobinformers "github.com/kubeflow/tf-operator/pkg/client/informers/externalversions"
	tfjobinformersv1alpha2 "github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/kubeflow/v1alpha2"
	"github.com/kubeflow/tf-operator/pkg/util/unstructured"
)

const (
	resyncPeriod = 30 * time.Second
)

var (
	errGetFromKey    = fmt.Errorf("Failed to get TFJob from key")
	errNotExists     = fmt.Errorf("The object is not found")
	errFailedMarshal = fmt.Errorf("Failed to marshal the object to TFJob")
)

func NewUnstructuredTFJobInformer(restConfig *restclientset.Config) tfjobinformersv1alpha2.TFJobInformer {
	dynClientPool := dynamic.NewDynamicClientPool(restConfig)
	dclient, err := dynClientPool.ClientForGroupVersionKind(controllerKind)
	if err != nil {
		panic(err)
	}
	resource := &metav1.APIResource{
		Name:         tfv1alpha2.Plural,
		SingularName: tfv1alpha2.Singular,
		Namespaced:   true,
		Group:        tfv1alpha2.GroupName,
		Version:      tfv1alpha2.GroupVersion,
	}
	informer := unstructured.NewTFJobInformer(
		resource,
		dclient,
		metav1.NamespaceAll,
		resyncPeriod,
		cache.Indexers{},
	)
	return informer
}

// NewTFJobInformer returns TFJobInformer from the given factory.
func (tc *TFJobController) NewTFJobInformer(tfJobInformerFactory tfjobinformers.SharedInformerFactory) tfjobinformersv1alpha2.TFJobInformer {
	return tfJobInformerFactory.Kubeflow().V1alpha2().TFJobs()
}

func (tc *TFJobController) getTFJobFromName(namespace, name string) (*tfv1alpha2.TFJob, error) {
	key := fmt.Sprintf("%s/%s", namespace, name)
	return tc.getTFJobFromKey(key)
}

func (tc *TFJobController) getTFJobFromKey(key string) (*tfv1alpha2.TFJob, error) {
	// Check if the key exists.
	obj, exists, err := tc.tfJobInformer.GetIndexer().GetByKey(key)
	if err != nil {
		log.Errorf("Failed to get TFJob '%s' from informer index: %+v", key, err)
		return nil, errGetFromKey
	}
	if !exists {
		// This happens after a tfjob was deleted, but the work queue still had an entry for it.
		return nil, errNotExists
	}

	tfjob, err := tfJobFromUnstructured(obj)
	if err != nil {
		return nil, err
	}
	return tfjob, nil
}

func tfJobFromUnstructured(obj interface{}) (*tfv1alpha2.TFJob, error) {
	// Check if the spec is valid.
	un, ok := obj.(*metav1unstructured.Unstructured)
	if !ok {
		log.Warn("The object in index is not an unstructured")
		return nil, errGetFromKey
	}
	var tfjob tfv1alpha2.TFJob
	err := runtime.DefaultUnstructuredConverter.FromUnstructured(un.Object, &tfjob)
	// This is a simple validation for TFJob to close
	// https://github.com/kubeflow/tf-operator/issues/641
	// TODO(gaocegege): Add more validation here.
	if err != nil || tfjob.Spec.TFReplicaSpecs == nil {
		return &tfjob, errFailedMarshal
	}
	return &tfjob, nil
}
<|endoftext|>


<commit_before>package backend

import (
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"syscall"
	"text/template"

	log "github.com/Sirupsen/logrus"
	"github.com/mchudgins/certMgr/pkg/healthz"
	"github.com/mchudgins/go-service-helper/pkg/loggingWriter"
	"github.com/prometheus/client_golang/prometheus"
)

type promWriter struct {
	w             http.ResponseWriter
	statusCode    int
	contentLength int
}

var (
	html = `
<!doctype html>
<html lang="en">
<head>
  <meta charset="utf-8">
  <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
  <title>Welcome to OpenShift</title>

</body>
</html>`

	httpRequestsReceived = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "httpRequestsReceived_total",
			Help: "Number of HTTP requests received.",
		},
		[]string{"url"},
	)
	httpRequestsProcessed = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "httpRequestsProcessed_total",
			Help: "Number of HTTP requests processed.",
		},
		[]string{"url", "status"},
	)
)

func init() {
	prometheus.MustRegister(httpRequestsReceived)
	prometheus.MustRegister(httpRequestsProcessed)
}

func NewPromWriter(w http.ResponseWriter) *promWriter {
	return &promWriter{w: w}
}

func (l *promWriter) Header() http.Header {
	return l.w.Header()
}

func (l *promWriter) Write(data []byte) (int, error) {
	l.contentLength += len(data)
	return l.w.Write(data)
}

func (l *promWriter) WriteHeader(status int) {
	l.statusCode = status
	l.w.WriteHeader(status)
}

func (l *promWriter) Length() int {
	return l.contentLength
}

func (l *promWriter) StatusCode() int {

	// if nobody set the status, but data has been written
	// then all must be well.
	if l.statusCode == 0 && l.contentLength > 0 {
		return http.StatusOK
	}

	return l.statusCode
}

func httpCounter(fn http.Handler) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		u := r.URL.Path
		httpRequestsReceived.With(prometheus.Labels{"url": u}).Inc()
		pw := NewPromWriter(w)
		defer func() {
			httpRequestsProcessed.With(prometheus.Labels{"url": u, "status": strconv.Itoa(pw.statusCode)}).Inc()
		}()

		fn.ServeHTTP(pw, r)
	}
}

func Run() error {
	log.Printf("backend.Run()")

	hostname, err := os.Hostname()
	if err != nil {
		log.Fatal(err)
	}

	// make a channel to listen on events,
	// then launch the servers.

	errc := make(chan error)

	// interrupt handler
	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
		errc <- fmt.Errorf("%s", <-c)
	}()

	// http server
	go func() {
		hc, err := healthz.NewConfig()
		healthzHandler, err := healthz.Handler(hc)
		if err != nil {
			log.Panic(err)
		}

		http.HandleFunc("/healthz", httpCounter(healthzHandler))
		http.Handle("/metrics", loggingWriter.HTTPLogrusLogger(httpCounter(prometheus.Handler())))
		http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
			httpRequestsReceived.With(prometheus.Labels{"url": "/"}).Inc()

			type data struct {
				Hostname string
			}

			tmp, err := template.New("/").Parse(html)
			if err != nil {
				log.WithError(err).WithField("template", "/").Errorf("Unable to parse template")
				return
			}

			err = tmp.Execute(w, data{Hostname: hostname})
			if err != nil {
				log.WithError(err).Error("Unable to execute template")
			}

			httpRequestsProcessed.With(prometheus.Labels{"url": "/", "status": "200"}).Inc()
		})

		log.Infof("HTTPS service listening on %s", ":8080")
		errc <- http.ListenAndServe(":8080", nil)
	}()

	// wait for somthin'
	log.Infof("exit: %s", <-errc)

	return nil
}
<commit_msg>refactored to use an http.Mux<commit_after>package backend

import (
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"syscall"
	"text/template"

	log "github.com/Sirupsen/logrus"
	"github.com/mchudgins/certMgr/pkg/healthz"
	"github.com/mchudgins/go-service-helper/pkg/loggingWriter"
	"github.com/prometheus/client_golang/prometheus"
)

type promWriter struct {
	w             http.ResponseWriter
	statusCode    int
	contentLength int
}

var (
	indexTemplate *template.Template
	html          = `
<!doctype html>
<html lang="en">
<head>
  <meta charset="utf-8">
  <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
  <title>Welcome to OpenShift</title>

</body>
</html>`

	httpRequestsReceived = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "httpRequestsReceived_total",
			Help: "Number of HTTP requests received.",
		},
		[]string{"url"},
	)
	httpRequestsProcessed = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "httpRequestsProcessed_total",
			Help: "Number of HTTP requests processed.",
		},
		[]string{"url", "status"},
	)
)

func init() {
	prometheus.MustRegister(httpRequestsReceived)
	prometheus.MustRegister(httpRequestsProcessed)

	indexTemplate = template.Must(template.New("/").Parse(html))
}

func NewPromWriter(w http.ResponseWriter) *promWriter {
	return &promWriter{w: w}
}

func (l *promWriter) Header() http.Header {
	return l.w.Header()
}

func (l *promWriter) Write(data []byte) (int, error) {
	l.contentLength += len(data)
	return l.w.Write(data)
}

func (l *promWriter) WriteHeader(status int) {
	l.statusCode = status
	l.w.WriteHeader(status)
}

func (l *promWriter) Length() int {
	return l.contentLength
}

func (l *promWriter) StatusCode() int {

	// if nobody set the status, but data has been written
	// then all must be well.
	if l.statusCode == 0 && l.contentLength > 0 {
		return http.StatusOK
	}

	return l.statusCode
}

func httpCounter(fn http.Handler) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		u := r.URL.Path
		httpRequestsReceived.With(prometheus.Labels{"url": u}).Inc()
		pw := NewPromWriter(w)
		defer func() {
			httpRequestsProcessed.With(prometheus.Labels{"url": u, "status": strconv.Itoa(pw.statusCode)}).Inc()
		}()

		fn.ServeHTTP(pw, r)
	}
}

func Run() error {
	log.Printf("backend.Run()")

	hostname, err := os.Hostname()
	if err != nil {
		log.Fatal(err)
	}

	// make a channel to listen on events,
	// then launch the servers.

	errc := make(chan error)

	// interrupt handler
	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
		errc <- fmt.Errorf("%s", <-c)
	}()

	// http server
	go func() {
		hc, err := healthz.NewConfig()
		healthzHandler, err := healthz.Handler(hc)
		if err != nil {
			log.Panic(err)
		}

		mux := http.NewServeMux()

		mux.Handle("/healthz", healthzHandler)
		mux.Handle("/metrics", prometheus.Handler())
		mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
			httpRequestsReceived.With(prometheus.Labels{"url": "/"}).Inc()

			type data struct {
				Hostname string
			}

			err = indexTemplate.Execute(w, data{Hostname: hostname})
			if err != nil {
				log.WithError(err).Error("Unable to execute template")
			}

			httpRequestsProcessed.With(prometheus.Labels{"url": "/", "status": "200"}).Inc()
		})

		log.Infof("HTTPS service listening on %s", ":8080")
		errc <- http.ListenAndServe(":8080", loggingWriter.HTTPLogrusLogger(httpCounter(mux)))
	}()

	// wait for somthin'
	log.Infof("exit: %s", <-errc)

	return nil
}
<|endoftext|>


<commit_before>package gqt_test

import (
	"fmt"
	"io/ioutil"
	"net"
	"os"

	"code.cloudfoundry.org/guardian/gqt/runner"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("Config File", func() {
	Describe("starting gdn server with the --config flag", func() {
		var (
			configFilePath string
			port           int
			client         *runner.RunningGarden
		)

		BeforeEach(func() {
			config.DebugIP = ""
			config.DebugPort = nil

			configFile := tempFile("", "gqt-config-file-tests")
			configFilePath = configFile.Name()

			port = 9080 + GinkgoParallelNode()
			fmt.Fprintf(configFile, `[server]
debug-bind-ip = 127.0.0.1
debug-bind-port = %d
`, port)
			Expect(configFile.Close()).To(Succeed())
			config.ConfigFilePath = configFilePath
		})

		JustBeforeEach(func() {
			client = runner.Start(config)
		})

		AfterEach(func() {
			if !config.StartupExpectedToFail {
				Expect(client.DestroyAndStop()).To(Succeed())
			} else {
				Expect(client.Cleanup()).To(Succeed())
			}
			Expect(os.RemoveAll(configFilePath)).To(Succeed())
		})

		It("starts the server with config values from the config file", func() {
			conn, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", port))
			Expect(err).NotTo(HaveOccurred())
			Expect(conn.Close()).To(Succeed())
		})

		Context("and also passing flags on the command line that override values in the config file", func() {
			BeforeEach(func() {
				config.DebugPort = intptr(7080 + GinkgoParallelNode())
			})

			It("starts the server with config values from the command line taking precedence over the config file", func() {
				conn, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", *config.DebugPort))
				Expect(err).NotTo(HaveOccurred())
				Expect(conn.Close()).To(Succeed())
			})
		})

		Context("when the provided config file is not a valid ini file", func() {
			BeforeEach(func() {
				config.StartupExpectedToFail = true
				Expect(ioutil.WriteFile(configFilePath, []byte("invalid-ini-file"), 0)).To(Succeed())
			})

			It("fails to start", func() {
				Eventually(client).Should(gexec.Exit(1))
			})
		})

		Context("when the provided config file path does not exist", func() {
			BeforeEach(func() {
				config.StartupExpectedToFail = true
				Expect(os.Remove(configFilePath)).To(Succeed())
			})

			It("fails to start", func() {
				Eventually(client).Should(gexec.Exit(1))
			})
		})

		Context("the configured checking interval is zero", func() {
			BeforeEach(func() {
				config.CPUThrottlingCheckInterval = uint64ptr(0)
				config.StartupExpectedToFail = true
			})

			It("fails to start", func() {
				Eventually(client).Should(gexec.Exit(1))
			})
		})

	})

})
<commit_msg>Skip the throttling interval validation test unless throttling is enabled<commit_after>package gqt_test

import (
	"fmt"
	"io/ioutil"
	"net"
	"os"

	"code.cloudfoundry.org/guardian/gqt/runner"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("Config File", func() {
	Describe("starting gdn server with the --config flag", func() {
		var (
			configFilePath string
			port           int
			client         *runner.RunningGarden
		)

		BeforeEach(func() {
			config.DebugIP = ""
			config.DebugPort = nil

			configFile := tempFile("", "gqt-config-file-tests")
			configFilePath = configFile.Name()

			port = 9080 + GinkgoParallelNode()
			fmt.Fprintf(configFile, `[server]
debug-bind-ip = 127.0.0.1
debug-bind-port = %d
`, port)
			Expect(configFile.Close()).To(Succeed())
			config.ConfigFilePath = configFilePath
		})

		JustBeforeEach(func() {
			client = runner.Start(config)
		})

		AfterEach(func() {
			if !config.StartupExpectedToFail {
				Expect(client.DestroyAndStop()).To(Succeed())
			} else {
				Expect(client.Cleanup()).To(Succeed())
			}
			Expect(os.RemoveAll(configFilePath)).To(Succeed())
		})

		It("starts the server with config values from the config file", func() {
			conn, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", port))
			Expect(err).NotTo(HaveOccurred())
			Expect(conn.Close()).To(Succeed())
		})

		Context("and also passing flags on the command line that override values in the config file", func() {
			BeforeEach(func() {
				config.DebugPort = intptr(7080 + GinkgoParallelNode())
			})

			It("starts the server with config values from the command line taking precedence over the config file", func() {
				conn, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", *config.DebugPort))
				Expect(err).NotTo(HaveOccurred())
				Expect(conn.Close()).To(Succeed())
			})
		})

		Context("when the provided config file is not a valid ini file", func() {
			BeforeEach(func() {
				config.StartupExpectedToFail = true
				Expect(ioutil.WriteFile(configFilePath, []byte("invalid-ini-file"), 0)).To(Succeed())
			})

			It("fails to start", func() {
				Eventually(client).Should(gexec.Exit(1))
			})
		})

		Context("when the provided config file path does not exist", func() {
			BeforeEach(func() {
				config.StartupExpectedToFail = true
				Expect(os.Remove(configFilePath)).To(Succeed())
			})

			It("fails to start", func() {
				Eventually(client).Should(gexec.Exit(1))
			})
		})

		Context("the configured checking interval is zero", func() {
			BeforeEach(func() {
				skipIfNotCPUThrottling()

				config.CPUThrottlingCheckInterval = uint64ptr(0)
				config.StartupExpectedToFail = true
			})

			It("fails to start", func() {
				Eventually(client).Should(gexec.Exit(1))
			})
		})
	})
})
<|endoftext|>


<commit_before>package hammy

import (
	"log"
	"time"
	"container/list"
)

// Buffer for reprocessed data
type SendBufferImpl struct {
	dataChan chan *IncomingData
	data *list.List
	// Timeout between sends
	sleepTime time.Duration
	rHandler RequestHandler
}

// Creates and initialize new SendBuffer
func NewSendBufferImpl(rh RequestHandler, cfg Config) (sb *SendBufferImpl) {
	sb = new(SendBufferImpl)
	sb.dataChan = make(chan *IncomingData)
	sb.data = list.New()
	sb.sleepTime = time.Duration(1000 * cfg.SendBuffer.SleepTime) * time.Millisecond
	sb.rHandler = rh
	return
}

// Locks and begins data processing
func (sb *SendBufferImpl) Listen() {
	timer := time.Tick(sb.sleepTime)

	for {
		select {
			case newData := <- sb.dataChan:
				sb.data.PushBack(newData)
			case <- timer:
				go sb.send(sb.data)
				sb.data = list.New()
		}
	}
}

// Enqueue data for reprocessing
func (sb *SendBufferImpl) Push(data *IncomingData) {
	sb.dataChan <- data
}

// Process detached data buffer
func (sb *SendBufferImpl) send(data *list.List) {
	if data.Len() == 0 {
		return
	}

	// 1) Merge list
	mData := make(IncomingData)
	// iterates over data list
	for e := data.Front(); e != nil; e = e.Next() {
		// e.Value is *IncomingData (panic otherwise)
		eData := e.Value.(*IncomingData)
		for hostK, hostV := range *eData {
			mV, hostFound := mData[hostK]
			if hostFound {
				for k, v := range hostV {
					iArr, kFound := mV[k]
					if kFound {
						newIArr := make([]IncomingValueData, len(v) + len(iArr))
						for i, vE := range iArr {
							newIArr[i] = vE
						}
						for j, vE := range v {
							newIArr[len(iArr) + j] = vE
						}
						mV[k] = newIArr
					} else {
						mV[k] = v
					}
				}
			} else {
				mData[hostK] = hostV
			}
		}
	}

	// 2) Process merged data
	errs := sb.rHandler.Handle(mData)

	if len(errs) > 0 {
		log.Printf("!! Error in SendBuffer: %v", errs) // FIXME
	}
}
<commit_msg>json errors in send_buffer<commit_after>package hammy

import (
	"log"
	"time"
	"container/list"
	"encoding/json"
)

// Buffer for reprocessed data
type SendBufferImpl struct {
	dataChan chan *IncomingData
	data *list.List
	// Timeout between sends
	sleepTime time.Duration
	rHandler RequestHandler
}

// Creates and initialize new SendBuffer
func NewSendBufferImpl(rh RequestHandler, cfg Config) (sb *SendBufferImpl) {
	sb = new(SendBufferImpl)
	sb.dataChan = make(chan *IncomingData)
	sb.data = list.New()
	sb.sleepTime = time.Duration(1000 * cfg.SendBuffer.SleepTime) * time.Millisecond
	sb.rHandler = rh
	return
}

// Locks and begins data processing
func (sb *SendBufferImpl) Listen() {
	timer := time.Tick(sb.sleepTime)

	for {
		select {
			case newData := <- sb.dataChan:
				sb.data.PushBack(newData)
			case <- timer:
				go sb.send(sb.data)
				sb.data = list.New()
		}
	}
}

// Enqueue data for reprocessing
func (sb *SendBufferImpl) Push(data *IncomingData) {
	sb.dataChan <- data
}

// Process detached data buffer
func (sb *SendBufferImpl) send(data *list.List) {
	if data.Len() == 0 {
		return
	}

	// 1) Merge list
	mData := make(IncomingData)
	// iterates over data list
	for e := data.Front(); e != nil; e = e.Next() {
		// e.Value is *IncomingData (panic otherwise)
		eData := e.Value.(*IncomingData)
		for hostK, hostV := range *eData {
			mV, hostFound := mData[hostK]
			if hostFound {
				for k, v := range hostV {
					iArr, kFound := mV[k]
					if kFound {
						newIArr := make([]IncomingValueData, len(v) + len(iArr))
						for i, vE := range iArr {
							newIArr[i] = vE
						}
						for j, vE := range v {
							newIArr[len(iArr) + j] = vE
						}
						mV[k] = newIArr
					} else {
						mV[k] = v
					}
				}
			} else {
				mData[hostK] = hostV
			}
		}
	}

	// 2) Process merged data
	errs := sb.rHandler.Handle(mData)

	if len(errs) > 0 {
		// FIXME
		errs_str := make(map[string]string)
		for k, v := range errs {
			errs_str[k] = v.Error()
		}
		b, err := json.Marshal(errs_str)
		if err == nil {
			log.Printf("!! Error in SendBuffer: %s", b)
		} else {
			log.Printf("!! Error in SendBuffer: <unable to dump errs: %#v>", err)
		}
	}
}
<|endoftext|>


<commit_before>package control

import (
	"context"
	"net"
	"net/http"
	"strings"
	"time"

	"github.com/rancher/remotedialer"
	"github.com/rancher/wrangler/pkg/kv"
	utilnet "k8s.io/apimachinery/pkg/util/net"
	"k8s.io/apiserver/pkg/endpoints/request"
	"k8s.io/kubernetes/cmd/kube-apiserver/app"
)

func setupTunnel() http.Handler {
	tunnelServer := remotedialer.New(authorizer, remotedialer.DefaultErrorWriter)
	setupProxyDialer(tunnelServer)
	return tunnelServer
}

func setupProxyDialer(tunnelServer *remotedialer.Server) {
	app.DefaultProxyDialerFn = utilnet.DialFunc(func(_ context.Context, network, address string) (net.Conn, error) {
		_, port, _ := net.SplitHostPort(address)
		addr := "127.0.0.1"
		if port != "" {
			addr += ":" + port
		}
		nodeName, _ := kv.Split(address, ":")
		return tunnelServer.Dial(nodeName, 15*time.Second, "tcp", addr)
	})
}

func authorizer(req *http.Request) (clientKey string, authed bool, err error) {
	user, ok := request.UserFrom(req.Context())
	if !ok {
		return "", false, nil
	}

	if strings.HasPrefix(user.GetName(), "system:node:") {
		return strings.TrimPrefix(user.GetName(), "system:node:"), true, nil
	}

	return "", false, nil
}
<commit_msg>If tunnel session does not exist fallback to default dialer<commit_after>package control

import (
	"context"
	"net"
	"net/http"
	"strings"
	"time"

	"github.com/rancher/remotedialer"
	"github.com/rancher/wrangler/pkg/kv"
	utilnet "k8s.io/apimachinery/pkg/util/net"
	"k8s.io/apiserver/pkg/endpoints/request"
	"k8s.io/kubernetes/cmd/kube-apiserver/app"
)

func setupTunnel() http.Handler {
	tunnelServer := remotedialer.New(authorizer, remotedialer.DefaultErrorWriter)
	setupProxyDialer(tunnelServer)
	return tunnelServer
}

func setupProxyDialer(tunnelServer *remotedialer.Server) {
	app.DefaultProxyDialerFn = utilnet.DialFunc(func(ctx context.Context, network, address string) (net.Conn, error) {
		_, port, _ := net.SplitHostPort(address)
		addr := "127.0.0.1"
		if port != "" {
			addr += ":" + port
		}
		nodeName, _ := kv.Split(address, ":")
		if tunnelServer.HasSession(nodeName) {
			return tunnelServer.Dial(nodeName, 15*time.Second, "tcp", addr)
		}
		var d net.Dialer
		return d.DialContext(ctx, network, address)
	})
}

func authorizer(req *http.Request) (clientKey string, authed bool, err error) {
	user, ok := request.UserFrom(req.Context())
	if !ok {
		return "", false, nil
	}

	if strings.HasPrefix(user.GetName(), "system:node:") {
		return strings.TrimPrefix(user.GetName(), "system:node:"), true, nil
	}

	return "", false, nil
}
<|endoftext|>


<commit_before>// Copyright 2017-2020 The Usacloud Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package core

import (
	"os"

	"github.com/sacloud/usacloud/pkg/cmd/root"

	"github.com/spf13/cobra"
)

// completionCmd represents the completion command
var completionCmd = &cobra.Command{
	Use:   "completion",
	Short: "Generates bash completion scripts",
	Long: `To load completion run

. <(usacloud completion)

To configure your bash shell to load completions for each session add to your bashrc

# ~/.bashrc or ~/.profile
. <(usacloud completion)
`,
	Hidden: true,
	RunE: func(cmd *cobra.Command, args []string) error {
		return root.Command.GenBashCompletion(os.Stdout)
	},
}

func init() {
	root.Command.AddCommand(completionCmd)
}
<commit_msg>Basic completion command<commit_after>// Copyright 2017-2020 The Usacloud Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package core

import (
	"fmt"
	"os"

	"github.com/sacloud/usacloud/pkg/cmd/root"

	"github.com/spf13/cobra"
)

// completionCmd represents the completion command
var completionCmd = &cobra.Command{
	Use:   "completion [bash|zsh|fish|powershell]",
	Short: "Generate completion script",
	Long: `To load completions:

Bash:

$ source <(usacloud completion bash)

# To load completions for each session, execute once:
Linux:
  $ usacloud completion bash > /etc/bash_completion.d/usacloud
MacOS:
  $ usacloud completion bash > /usr/local/etc/bash_completion.d/usacloud

Zsh:

# If shell completion is not already enabled in your environment you will need
# to enable it.  You can execute the following once:

$ echo "autoload -U compinit; compinit" >> ~/.zshrc

# To load completions for each session, execute once:
$ usacloud completion zsh > "${fpath[1]}/_usacloud"

# You will need to start a new shell for this setup to take effect.

Fish:

$ usacloud completion fish | source

# To load completions for each session, execute once:
$ usacloud completion fish > ~/.config/fish/completions/usacloud.fish
`,
	DisableFlagsInUseLine: true,
	ValidArgs:             []string{"bash", "zsh", "fish", "powershell"},
	Args:                  cobra.ExactValidArgs(1),
	Run: func(cmd *cobra.Command, args []string) {
		var err error
		switch args[0] {
		case "bash":
			err = cmd.Root().GenBashCompletion(os.Stdout)
		case "zsh":
			err = cmd.Root().GenZshCompletion(os.Stdout)
		case "fish":
			err = cmd.Root().GenFishCompletion(os.Stdout, true)
		case "powershell":
			err = cmd.Root().GenPowerShellCompletion(os.Stdout)
		}
		if err != nil {
			fmt.Fprintln(os.Stderr, err) // nolint
		}
	},
}

func init() {
	root.Command.AddCommand(completionCmd)
}
<|endoftext|>


<commit_before>package prices

import (
	"bytes"
	"io"
	"io/ioutil"
	"net/http"

	"github.com/antonholmquist/jason"
	log "github.com/mozilla-services/reaper/reaperlog"
)

const Ec2PricingUrl = "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEC2/current/index.json"

type PricesMap map[string]map[string]string

var regions = map[string]string{
	"US West (N. California)":   "us-west-1",
	"US West (Oregon)":          "us-west-2",
	"EU (Ireland)":              "eu-west-1",
	"EU (Frankfurt)":            "eu-central-1",
	"Asia Pacific (Seoul)":      "ap-northeast-2",
	"Asia Pacific (Sydney)":     "ap-southeast-2",
	"US East (N. Virginia)":     "us-east-1",
	"South America (Sao Paulo)": "sa-east-1",
	"Asia Pacific (Singapore)":  "ap-southeast-1",
	"Asia Pacific (Tokyo)":      "ap-northeast-1",
	"AWS GovCloud (US)":         "AWS GovCloud (US)",
}

func GetPricesMapFromFile(filename string) (PricesMap, error) {
	if filename == "" {
		return PricesMap{}, nil
	}
	bs, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	return populatePricesMap(bytes.NewReader(bs))
}

func DownloadPricesMap(url string) (PricesMap, error) {
	if url == "" {
		return PricesMap{}, nil
	}
	res, err := http.Get(url)
	if err != nil {
		log.Error(err.Error())
	}
	defer res.Body.Close()
	return populatePricesMap(res.Body)
}

func populatePricesMap(r io.Reader) (PricesMap, error) {
	pricesMap := make(PricesMap)
	for _, region := range regions {
		pricesMap[region] = make(map[string]string)
	}

	// initialize inner map
	decoded, err := jason.NewObjectFromReader(r)
	if err != nil {
		return PricesMap{}, err
	}
	products, err := decoded.GetObject("products")
	if err != nil {
		return PricesMap{}, err
	}
	for _, skuObject := range products.Map() {
		individualSkuObject, err := skuObject.Object()
		if err != nil {
			return PricesMap{}, err
		}
		sku, err := individualSkuObject.Map()["sku"].String()
		if err != nil {
			return PricesMap{}, err
		}
		attributes, err := individualSkuObject.Map()["attributes"].Object()
		if err != nil {
			return PricesMap{}, err
		}
		if instanceType, ok := attributes.Map()["instanceType"]; !ok {
			// if it's not an instance, skip it
			continue
		} else {
			instanceTypeString, err := instanceType.String()
			if err != nil {
				return PricesMap{}, err
			}
			location, err := attributes.Map()["location"].String()
			if err != nil {
				return PricesMap{}, err
			}
			// get pricing information from a separate part of the json file by sku
			priceSkuObject, err := decoded.GetObject("terms", "OnDemand", sku)
			if err != nil {
				return PricesMap{}, err
			}
			for _, specificSku := range priceSkuObject.Map() {
				specificSkuObject, err := specificSku.Object()
				if err != nil {
					return PricesMap{}, err
				}
				priceDimensionsObject, err := specificSkuObject.Map()["priceDimensions"].Object()
				if err != nil {
					return PricesMap{}, err
				}
				for _, specificPriceDimensions := range priceDimensionsObject.Map() {
					specificPriceDimensionsObject, err := specificPriceDimensions.Object()
					if err != nil {
						return PricesMap{}, err
					}
					pricePerUnitObject, err := specificPriceDimensionsObject.Map()["pricePerUnit"].Object()
					if err != nil {
						return PricesMap{}, err
					}
					USD, err := pricePerUnitObject.Map()["USD"].String()
					if err != nil {
						return PricesMap{}, err
					}
					pricesMap[regions[location]][instanceTypeString] = USD
				}
			}
		}
	}
	return pricesMap, nil
}
<commit_msg>insecure tls to fix docker certs headache<commit_after>package prices

import (
	"bytes"
	"crypto/tls"
	"io"
	"io/ioutil"
	"net/http"

	"github.com/antonholmquist/jason"
	log "github.com/mozilla-services/reaper/reaperlog"
)

const Ec2PricingUrl = "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEC2/current/index.json"

type PricesMap map[string]map[string]string

var regions = map[string]string{
	"US West (N. California)":   "us-west-1",
	"US West (Oregon)":          "us-west-2",
	"EU (Ireland)":              "eu-west-1",
	"EU (Frankfurt)":            "eu-central-1",
	"Asia Pacific (Seoul)":      "ap-northeast-2",
	"Asia Pacific (Sydney)":     "ap-southeast-2",
	"US East (N. Virginia)":     "us-east-1",
	"South America (Sao Paulo)": "sa-east-1",
	"Asia Pacific (Singapore)":  "ap-southeast-1",
	"Asia Pacific (Tokyo)":      "ap-northeast-1",
	"AWS GovCloud (US)":         "AWS GovCloud (US)",
}

func GetPricesMapFromFile(filename string) (PricesMap, error) {
	if filename == "" {
		return PricesMap{}, nil
	}
	bs, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	return populatePricesMap(bytes.NewReader(bs))
}

func DownloadPricesMap(url string) (PricesMap, error) {
	if url == "" {
		return PricesMap{}, nil
	}

	// TODO: working TLS
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	client := &http.Client{Transport: tr}

	res, err := client.Get(url)
	if err != nil {
		log.Error(err.Error())
	}
	defer res.Body.Close()
	return populatePricesMap(res.Body)
}

func populatePricesMap(r io.Reader) (PricesMap, error) {
	pricesMap := make(PricesMap)
	for _, region := range regions {
		pricesMap[region] = make(map[string]string)
	}

	// initialize inner map
	decoded, err := jason.NewObjectFromReader(r)
	if err != nil {
		return PricesMap{}, err
	}
	products, err := decoded.GetObject("products")
	if err != nil {
		return PricesMap{}, err
	}
	for _, skuObject := range products.Map() {
		individualSkuObject, err := skuObject.Object()
		if err != nil {
			return PricesMap{}, err
		}
		sku, err := individualSkuObject.Map()["sku"].String()
		if err != nil {
			return PricesMap{}, err
		}
		attributes, err := individualSkuObject.Map()["attributes"].Object()
		if err != nil {
			return PricesMap{}, err
		}
		if instanceType, ok := attributes.Map()["instanceType"]; !ok {
			// if it's not an instance, skip it
			continue
		} else {
			instanceTypeString, err := instanceType.String()
			if err != nil {
				return PricesMap{}, err
			}
			location, err := attributes.Map()["location"].String()
			if err != nil {
				return PricesMap{}, err
			}
			// get pricing information from a separate part of the json file by sku
			priceSkuObject, err := decoded.GetObject("terms", "OnDemand", sku)
			if err != nil {
				return PricesMap{}, err
			}
			for _, specificSku := range priceSkuObject.Map() {
				specificSkuObject, err := specificSku.Object()
				if err != nil {
					return PricesMap{}, err
				}
				priceDimensionsObject, err := specificSkuObject.Map()["priceDimensions"].Object()
				if err != nil {
					return PricesMap{}, err
				}
				for _, specificPriceDimensions := range priceDimensionsObject.Map() {
					specificPriceDimensionsObject, err := specificPriceDimensions.Object()
					if err != nil {
						return PricesMap{}, err
					}
					pricePerUnitObject, err := specificPriceDimensionsObject.Map()["pricePerUnit"].Object()
					if err != nil {
						return PricesMap{}, err
					}
					USD, err := pricePerUnitObject.Map()["USD"].String()
					if err != nil {
						return PricesMap{}, err
					}
					pricesMap[regions[location]][instanceTypeString] = USD
				}
			}
		}
	}
	return pricesMap, nil
}
<|endoftext|>


<commit_before>package libkbfs

import (
	"fmt"
	"runtime"
	"sync"
	"testing"

	keybase1 "github.com/keybase/client/protocol/go"
	"github.com/keybase/kbfs/util"
)

type RWChannelCounter struct {
	lock   sync.Mutex
	rwc    util.RWScheduler
	rcount int
	wcount int
}

func NewRWChannelCounter(bufSize int) *RWChannelCounter {
	return &RWChannelCounter{rwc: util.NewRWChannel(bufSize)}
}

func (rwc *RWChannelCounter) QueueReadReq(rreq func()) {
	rwc.lock.Lock()
	rwc.rcount++
	rwc.lock.Unlock()
	rwc.rwc.QueueReadReq(rreq)
}

func (rwc *RWChannelCounter) QueueWriteReq(wreq func()) {
	rwc.lock.Lock()
	rwc.wcount++
	rwc.lock.Unlock()
	rwc.rwc.QueueWriteReq(wreq)
}

func (rwc *RWChannelCounter) Shutdown() chan struct{} {
	return rwc.rwc.Shutdown()
}

type MDOpsConcurTest struct {
	uid   keybase1.UID
	enter chan struct{}
	start chan struct{}
}

func NewMDOpsConcurTest(uid keybase1.UID) *MDOpsConcurTest {
	return &MDOpsConcurTest{
		uid:   uid,
		enter: make(chan struct{}),
		start: make(chan struct{}),
	}
}

func (m *MDOpsConcurTest) GetAtHandle(handle *DirHandle) (
	*RootMetadata, error) {
	return nil, fmt.Errorf("Not supported")
}

func (m *MDOpsConcurTest) Get(id DirID) (*RootMetadata, error) {
	_, ok := <-m.enter
	if !ok {
		// Only one caller should ever get here
		return nil, fmt.Errorf("More than one caller to Get()!")
	}
	<-m.start
	dh := NewDirHandle()
	dh.Writers = append(dh.Writers, m.uid)
	return NewRootMetadata(dh, id), nil
}

func (m *MDOpsConcurTest) GetAtID(id DirID, mdID MdID) (*RootMetadata, error) {
	return nil, fmt.Errorf("Not supported")
}

func (m *MDOpsConcurTest) Put(id DirID, md *RootMetadata) error {
	return nil
}

func (m *MDOpsConcurTest) GetFavorites() ([]DirID, error) {
	return []DirID{}, nil
}

func kbfsOpsConcurInit(t *testing.T, users ...string) (Config, keybase1.UID) {
	config := MakeTestConfigOrBust(false, users...)

	loggedInUser, err := config.KBPKI().GetLoggedInUser()
	if err != nil {
		t.Fatal(err)
	}

	return config, loggedInUser
}

// Test that only one of two concurrent GetRootMD requests can end up
// fetching the MD from the server.  The second one should wait, and
// then get it from the MD cache.
func TestKBFSOpsConcurDoubleMDGet(t *testing.T) {
	config, uid := kbfsOpsConcurInit(t, "test_user")
	defer config.KBFSOps().(*KBFSOpsStandard).Shutdown()
	m := NewMDOpsConcurTest(uid)
	config.SetMDOps(m)

	n := 10
	c := make(chan error, n)
	dir := DirID{0}
	rwc := NewRWChannelCounter(n)
	config.KBFSOps().(*KBFSOpsStandard).dirRWChans.chans[dir] = rwc
	for i := 0; i < n; i++ {
		go func() {
			_, _, _, err := config.KBFSOps().GetRootPath(dir)
			c <- err
		}()
	}
	// wait until at least the first one started
	m.enter <- struct{}{}
	close(m.enter)
	// make sure that the second goroutine has also started its write
	// call, and thus must be queued behind the first one (since we
	// are guaranteed the first one is currently running, and
	// RWChannel only allows one write at a time).
	for {
		rwc.lock.Lock()
		rc := rwc.rcount
		wc := rwc.wcount
		rwc.lock.Unlock()
		if rc == n && wc >= 2 {
			break
		} else {
			runtime.Gosched()
		}
	}
	// Now let the first one complete.  The second one should find the
	// MD in the cache, and thus never call MDOps.Get().
	m.start <- struct{}{}
	close(m.start)
	for i := 0; i < n; i++ {
		err := <-c
		if err != nil {
			t.Errorf("Got an error doing concurrent MD gets: err=(%s)", err)
		}
	}
}
<commit_msg>kbfs_ops_concur_test: fix compile breakage<commit_after>package libkbfs

import (
	"fmt"
	"runtime"
	"sync"
	"testing"

	keybase1 "github.com/keybase/client/protocol/go"
	"github.com/keybase/kbfs/util"
)

type RWChannelCounter struct {
	lock   sync.Mutex
	rwc    util.RWScheduler
	rcount int
	wcount int
}

func NewRWChannelCounter(bufSize int) *RWChannelCounter {
	return &RWChannelCounter{rwc: util.NewRWChannel(bufSize)}
}

func (rwc *RWChannelCounter) QueueReadReq(rreq func()) {
	rwc.lock.Lock()
	rwc.rcount++
	rwc.lock.Unlock()
	rwc.rwc.QueueReadReq(rreq)
}

func (rwc *RWChannelCounter) QueueWriteReq(wreq func()) {
	rwc.lock.Lock()
	rwc.wcount++
	rwc.lock.Unlock()
	rwc.rwc.QueueWriteReq(wreq)
}

func (rwc *RWChannelCounter) Shutdown() chan struct{} {
	return rwc.rwc.Shutdown()
}

type MDOpsConcurTest struct {
	uid   keybase1.UID
	enter chan struct{}
	start chan struct{}
}

func NewMDOpsConcurTest(uid keybase1.UID) *MDOpsConcurTest {
	return &MDOpsConcurTest{
		uid:   uid,
		enter: make(chan struct{}),
		start: make(chan struct{}),
	}
}

func (m *MDOpsConcurTest) GetAtHandle(handle *DirHandle) (
	*RootMetadata, error) {
	return nil, fmt.Errorf("Not supported")
}

func (m *MDOpsConcurTest) Get(id DirID) (*RootMetadata, error) {
	_, ok := <-m.enter
	if !ok {
		// Only one caller should ever get here
		return nil, fmt.Errorf("More than one caller to Get()!")
	}
	<-m.start
	dh := NewDirHandle()
	dh.Writers = append(dh.Writers, m.uid)
	return NewRootMetadata(dh, id), nil
}

func (m *MDOpsConcurTest) GetAtID(id DirID, mdID MdID) (*RootMetadata, error) {
	return nil, fmt.Errorf("Not supported")
}

func (m *MDOpsConcurTest) Put(id DirID, md *RootMetadata) error {
	return nil
}

func (m *MDOpsConcurTest) GetFavorites() ([]DirID, error) {
	return []DirID{}, nil
}

func kbfsOpsConcurInit(t *testing.T, users ...string) (Config, keybase1.UID) {
	config := MakeTestConfigOrBust(t, false, users...)

	loggedInUser, err := config.KBPKI().GetLoggedInUser()
	if err != nil {
		t.Fatal(err)
	}

	return config, loggedInUser
}

// Test that only one of two concurrent GetRootMD requests can end up
// fetching the MD from the server.  The second one should wait, and
// then get it from the MD cache.
func TestKBFSOpsConcurDoubleMDGet(t *testing.T) {
	config, uid := kbfsOpsConcurInit(t, "test_user")
	defer config.KBFSOps().(*KBFSOpsStandard).Shutdown()
	m := NewMDOpsConcurTest(uid)
	config.SetMDOps(m)

	n := 10
	c := make(chan error, n)
	dir := DirID{0}
	rwc := NewRWChannelCounter(n)
	config.KBFSOps().(*KBFSOpsStandard).dirRWChans.chans[dir] = rwc
	for i := 0; i < n; i++ {
		go func() {
			_, _, _, err := config.KBFSOps().GetRootPath(dir)
			c <- err
		}()
	}
	// wait until at least the first one started
	m.enter <- struct{}{}
	close(m.enter)
	// make sure that the second goroutine has also started its write
	// call, and thus must be queued behind the first one (since we
	// are guaranteed the first one is currently running, and
	// RWChannel only allows one write at a time).
	for {
		rwc.lock.Lock()
		rc := rwc.rcount
		wc := rwc.wcount
		rwc.lock.Unlock()
		if rc == n && wc >= 2 {
			break
		} else {
			runtime.Gosched()
		}
	}
	// Now let the first one complete.  The second one should find the
	// MD in the cache, and thus never call MDOps.Get().
	m.start <- struct{}{}
	close(m.start)
	for i := 0; i < n; i++ {
		err := <-c
		if err != nil {
			t.Errorf("Got an error doing concurrent MD gets: err=(%s)", err)
		}
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"errors"
	"fmt"
	"io"

	"github.com/apprenda/kismatic-platform/pkg/inspector/check"
	"github.com/apprenda/kismatic-platform/pkg/inspector/rule"
	"github.com/spf13/cobra"
)

// NewCmdLocal returns the "local" command
func NewCmdLocal(out io.Writer) *cobra.Command {
	var outputType string
	var nodeRole string
	cmd := &cobra.Command{
		Use:   "local",
		Short: "run the inspector checks locally",
		RunE: func(cmd *cobra.Command, args []string) error {
			return runLocal(out, outputType, nodeRole)
		},
	}
	cmd.Flags().StringVarP(&outputType, "output", "o", "table", "set the result output type. Options are 'json', 'table'")
	cmd.Flags().StringVar(&nodeRole, "node-role", "", "the node's role in the cluster. Options are 'etcd', 'master', 'worker'")
	return cmd
}

func runLocal(out io.Writer, outputType, nodeRole string) error {
	if nodeRole == "" {
		return fmt.Errorf("node role is required")
	}
	if nodeRole != "etcd" && nodeRole != "master" && nodeRole != "worker" {
		return fmt.Errorf("%s is not a valid node role", nodeRole)
	}
	// Get the printer
	var printResults resultPrinter
	switch outputType {
	case "json":
		printResults = printResultsAsJSON
	case "table":
		printResults = printResultsAsTable
	default:
		return fmt.Errorf("output type %q not supported", outputType)
	}
	distro, err := check.DetectDistro()
	if err != nil {
		return fmt.Errorf("error running checks locally: %v", err)
	}
	_, err = check.NewPackageManager(distro)
	if err != nil {
		return err
	}
	pkgMgr, err := check.NewPackageManager(distro)
	if err != nil {
		return err
	}
	m := rule.DefaultRules()
	e := rule.Engine{
		RuleCheckMapper: rule.DefaultCheckMapper{
			PackageManager: pkgMgr,
		},
	}
	labels := []string{string(distro), nodeRole}
	results, err := e.ExecuteRules(m, labels)
	if err != nil {
		return fmt.Errorf("error running local rules: %v", err)
	}
	printResults(out, results)
	for _, r := range results {
		if !r.Success {
			return errors.New("inspector found checks that failed")
		}
	}
	return nil
}
<commit_msg>KIS-71: Add ability to alter rule set when running in local mode<commit_after>package cmd

import (
	"errors"
	"fmt"
	"io"

	"github.com/apprenda/kismatic-platform/pkg/inspector/check"
	"github.com/apprenda/kismatic-platform/pkg/inspector/rule"
	"github.com/spf13/cobra"
)

type localOpts struct {
	outputType string
	nodeRole   string
	rulesFile  string
}

// NewCmdLocal returns the "local" command
func NewCmdLocal(out io.Writer) *cobra.Command {
	opts := localOpts{}
	cmd := &cobra.Command{
		Use:   "local",
		Short: "run the inspector checks locally",
		RunE: func(cmd *cobra.Command, args []string) error {
			return runLocal(out, opts)
		},
	}
	cmd.Flags().StringVarP(&opts.outputType, "output", "o", "table", "set the result output type. Options are 'json', 'table'")
	cmd.Flags().StringVar(&opts.nodeRole, "node-role", "", "the node's role in the cluster. Options are 'etcd', 'master', 'worker'")
	cmd.Flags().StringVarP(&opts.rulesFile, "file", "f", "", "the path to an inspector rules file. If blank, the inspector uses the default rules")
	return cmd
}

func runLocal(out io.Writer, opts localOpts) error {
	nodeRole := opts.nodeRole
	if nodeRole == "" {
		return fmt.Errorf("node role is required")
	}
	if nodeRole != "etcd" && nodeRole != "master" && nodeRole != "worker" {
		return fmt.Errorf("%s is not a valid node role", nodeRole)
	}
	// Get the printer
	var printResults resultPrinter
	switch opts.outputType {
	case "json":
		printResults = printResultsAsJSON
	case "table":
		printResults = printResultsAsTable
	default:
		return fmt.Errorf("output type %q not supported", opts.outputType)
	}
	// Gather rules
	var rules []rule.Rule
	var err error
	if opts.rulesFile != "" {
		rules, err = rule.ReadFromFile(opts.rulesFile)
		if err != nil {
			return err
		}
	} else {
		rules = rule.DefaultRules()
	}
	// Set up engine dependencies
	distro, err := check.DetectDistro()
	if err != nil {
		return fmt.Errorf("error running checks locally: %v", err)
	}
	_, err = check.NewPackageManager(distro)
	if err != nil {
		return err
	}
	pkgMgr, err := check.NewPackageManager(distro)
	if err != nil {
		return err
	}
	// Create rule engine
	e := rule.Engine{
		RuleCheckMapper: rule.DefaultCheckMapper{
			PackageManager: pkgMgr,
		},
	}
	labels := []string{string(distro), nodeRole}
	results, err := e.ExecuteRules(rules, labels)
	if err != nil {
		return fmt.Errorf("error running local rules: %v", err)
	}
	printResults(out, results)
	for _, r := range results {
		if !r.Success {
			return errors.New("inspector found checks that failed")
		}
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package hyper

const (
	StatusRunning = "running"
	StatusPending = "pending"
	StatusFailed  = "failed"
	StatusSuccess = "succeeded"
)

type HyperImage struct {
	repository  string
	tag         string
	imageID     string
	createdAt   int64
	virtualSize int64
}

// Container JSON Data Structure
type ContainerPort struct {
	Name          string `json:"name"`
	HostPort      int    `json:"hostPort"`
	ContainerPort int    `json:"containerPort"`
	Protocol      string `json:"protocol"`
	HostIP        string `json:"hostIP"`
}

type EnvironmentVar struct {
	Env   string `json:"env"`
	Value string `json:"value"`
}

type VolumeMount struct {
	Name      string `json:"name"`
	ReadOnly  bool   `json:"readOnly"`
	MountPath string `json:"mountPath"`
}

type WaitingStatus struct {
	Reason string `json:"reason"`
}

type RunningStatus struct {
	StartedAt string `json:"startedAt"`
}

type TermStatus struct {
	ExitCode   int    `json:"exitCode"`
	Reason     string `json:"reason"`
	Message    string `json:"message"`
	StartedAt  string `json:"startedAt"`
	FinishedAt string `json:"finishedAt"`
}

type ContainerStatus struct {
	Name        string        `json:"name"`
	ContainerID string        `json:"containerID"`
	Phase       string        `json:"phase"`
	Waiting     WaitingStatus `json:"waiting"`
	Running     RunningStatus `json:"running"`
	Terminated  TermStatus    `json:"terminated"`
}

// Pod JSON Data Structure
type Container struct {
	Name            string           `json:"name"`
	ContainerID     string           `json:"containerID"`
	PodID           string           `json:"podID"`
	Image           string           `json:"image"`
	ImageID         string           `json:"imageID"`
	Commands        []string         `json:"commands"`
	Args            []string         `json:"args"`
	Workdir         string           `json:"workingDir"`
	Ports           []ContainerPort  `json:"ports"`
	Environment     []EnvironmentVar `json:"env"`
	Volume          []VolumeMount    `json:"volumeMounts"`
	ImagePullPolicy string           `json:"imagePullPolicy"`
}

type RBDVolumeSource struct {
	Monitors []string `json:"monitors"`
	Image    string   `json:"image"`
	FsType   string   `json:"fsType"`
	Pool     string   `json:"pool"`
	User     string   `json:"user"`
	Keyring  string   `json:"keyring"`
	ReadOnly bool     `json:"readOnly"`
}

type PodVolume struct {
	Name     string          `json:"name"`
	HostPath string          `json:"source"`
	Driver   string          `json:"driver"`
	Rbd      RBDVolumeSource `json:"rbd"`
}

type PodSpec struct {
	Volumes    []PodVolume       `json:"volumes"`
	Containers []Container       `json:"containers"`
	Labels     map[string]string `json:"labels"`
	Vcpu       int               `json:"vcpu"`
	Memory     int               `json:"memory"`
}

type PodStatus struct {
	Phase      string            `json:"phase"`
	Message    string            `json:"message"`
	Reason     string            `json:"reason"`
	HostIP     string            `json:"hostIP"`
	PodIP      []string          `json:"podIP"`
	StartTime  string            `json:"startTime"`
	FinishTime string            `json:"finishTime"`
	Status     []ContainerStatus `json:"containerStatus"`
}

type PodInfo struct {
	Kind       string    `json:"kind"`
	ApiVersion string    `json:"apiVersion"`
	Vm         string    `json:"vm"`
	Spec       PodSpec   `json:"spec"`
	Status     PodStatus `json:"status"`
}

type HyperPod struct {
	PodID   string
	PodName string
	VmName  string
	Status  string
	PodInfo PodInfo
}

type HyperContainer struct {
	containerID string
	name        string
	podID       string
	status      string
}

type HyperServiceBackend struct {
	HostIP   string `json:"hostip"`
	HostPort int    `json:"hostport"`
}

type HyperService struct {
	ServiceIP   string                `json:"serviceip"`
	ServicePort int                   `json:"serviceport"`
	Protocol    string                `json:"protocol"`
	Hosts       []HyperServiceBackend `json:"hosts"`
}
<commit_msg>Fix hyper service<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package hyper

const (
	StatusRunning = "running"
	StatusPending = "pending"
	StatusFailed  = "failed"
	StatusSuccess = "succeeded"
)

type HyperImage struct {
	repository  string
	tag         string
	imageID     string
	createdAt   int64
	virtualSize int64
}

// Container JSON Data Structure
type ContainerPort struct {
	Name          string `json:"name"`
	HostPort      int    `json:"hostPort"`
	ContainerPort int    `json:"containerPort"`
	Protocol      string `json:"protocol"`
	HostIP        string `json:"hostIP"`
}

type EnvironmentVar struct {
	Env   string `json:"env"`
	Value string `json:"value"`
}

type VolumeMount struct {
	Name      string `json:"name"`
	ReadOnly  bool   `json:"readOnly"`
	MountPath string `json:"mountPath"`
}

type WaitingStatus struct {
	Reason string `json:"reason"`
}

type RunningStatus struct {
	StartedAt string `json:"startedAt"`
}

type TermStatus struct {
	ExitCode   int    `json:"exitCode"`
	Reason     string `json:"reason"`
	Message    string `json:"message"`
	StartedAt  string `json:"startedAt"`
	FinishedAt string `json:"finishedAt"`
}

type ContainerStatus struct {
	Name        string        `json:"name"`
	ContainerID string        `json:"containerID"`
	Phase       string        `json:"phase"`
	Waiting     WaitingStatus `json:"waiting"`
	Running     RunningStatus `json:"running"`
	Terminated  TermStatus    `json:"terminated"`
}

// Pod JSON Data Structure
type Container struct {
	Name            string           `json:"name"`
	ContainerID     string           `json:"containerID"`
	PodID           string           `json:"podID"`
	Image           string           `json:"image"`
	ImageID         string           `json:"imageID"`
	Commands        []string         `json:"commands"`
	Args            []string         `json:"args"`
	Workdir         string           `json:"workingDir"`
	Ports           []ContainerPort  `json:"ports"`
	Environment     []EnvironmentVar `json:"env"`
	Volume          []VolumeMount    `json:"volumeMounts"`
	ImagePullPolicy string           `json:"imagePullPolicy"`
}

type RBDVolumeSource struct {
	Monitors []string `json:"monitors"`
	Image    string   `json:"image"`
	FsType   string   `json:"fsType"`
	Pool     string   `json:"pool"`
	User     string   `json:"user"`
	Keyring  string   `json:"keyring"`
	ReadOnly bool     `json:"readOnly"`
}

type PodVolume struct {
	Name     string          `json:"name"`
	HostPath string          `json:"source"`
	Driver   string          `json:"driver"`
	Rbd      RBDVolumeSource `json:"rbd"`
}

type PodSpec struct {
	Volumes    []PodVolume       `json:"volumes"`
	Containers []Container       `json:"containers"`
	Labels     map[string]string `json:"labels"`
	Vcpu       int               `json:"vcpu"`
	Memory     int               `json:"memory"`
}

type PodStatus struct {
	Phase      string            `json:"phase"`
	Message    string            `json:"message"`
	Reason     string            `json:"reason"`
	HostIP     string            `json:"hostIP"`
	PodIP      []string          `json:"podIP"`
	StartTime  string            `json:"startTime"`
	FinishTime string            `json:"finishTime"`
	Status     []ContainerStatus `json:"containerStatus"`
}

type PodInfo struct {
	Kind       string    `json:"kind"`
	ApiVersion string    `json:"apiVersion"`
	Vm         string    `json:"vm"`
	Spec       PodSpec   `json:"spec"`
	Status     PodStatus `json:"status"`
}

type HyperPod struct {
	PodID   string
	PodName string
	VmName  string
	Status  string
	PodInfo PodInfo
}

type HyperContainer struct {
	containerID string
	name        string
	podID       string
	status      string
}

type HyperServiceBackend struct {
	HostIP   string `json:"hostIP"`
	HostPort int    `json:"hostPort"`
}

type HyperService struct {
	ServiceIP   string                `json:"serviceIP"`
	ServicePort int                   `json:"servicePort"`
	Protocol    string                `json:"protocol"`
	Hosts       []HyperServiceBackend `json:"hosts"`
}
<|endoftext|>


<commit_before>// Copyright 2014 Wandoujia Inc. All Rights Reserved.
// Licensed under the MIT (MIT-LICENSE.txt) license.

package models

import (
	"encoding/json"
	"fmt"
	"strconv"
	"strings"

	"github.com/wandoulabs/codis/pkg/utils"
	"github.com/wandoulabs/codis/pkg/utils/errors"
	"github.com/wandoulabs/codis/pkg/utils/log"
	"github.com/wandoulabs/zkhelper"
)

const (
	SERVER_TYPE_MASTER  string = "master"
	SERVER_TYPE_SLAVE   string = "slave"
	SERVER_TYPE_OFFLINE string = "offline"
)

// redis server instance
type Server struct {
	Type    string `json:"type"`
	GroupId int    `json:"group_id"`
	Addr    string `json:"addr"`
}

// redis server group
type ServerGroup struct {
	Id          int       `json:"id"`
	ProductName string    `json:"product_name"`
	Servers     []*Server `json:"servers"`
}

func (self *Server) String() string {
	b, _ := json.MarshalIndent(self, "", "  ")
	return string(b)
}

func (self *ServerGroup) String() string {
	b, _ := json.MarshalIndent(self, "", "  ")
	return string(b) + "\n"
}

func GetServer(zkConn zkhelper.Conn, zkPath string) (*Server, error) {
	data, _, err := zkConn.Get(zkPath)
	if err != nil {
		return nil, errors.Trace(err)
	}
	srv := Server{}
	if err := json.Unmarshal(data, &srv); err != nil {
		return nil, errors.Trace(err)
	}
	return &srv, nil
}

func NewServer(serverType string, addr string) *Server {
	return &Server{
		Type:    serverType,
		GroupId: INVALID_ID,
		Addr:    addr,
	}
}

func NewServerGroup(productName string, id int) *ServerGroup {
	return &ServerGroup{
		Id:          id,
		ProductName: productName,
	}
}

func GroupExists(zkConn zkhelper.Conn, productName string, groupId int) (bool, error) {
	zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d", productName, groupId)
	exists, _, err := zkConn.Exists(zkPath)
	if err != nil {
		return false, errors.Trace(err)
	}
	return exists, nil
}

func GetGroup(zkConn zkhelper.Conn, productName string, groupId int) (*ServerGroup, error) {
	exists, err := GroupExists(zkConn, productName, groupId)
	if err != nil {
		return nil, errors.Trace(err)
	}
	if !exists {
		return nil, errors.Errorf("group %d is not found", groupId)
	}

	group := &ServerGroup{
		ProductName: productName,
		Id:          groupId,
	}

	group.Servers, err = group.GetServers(zkConn)
	if err != nil {
		return nil, errors.Trace(err)
	}
	return group, nil
}

func ServerGroups(zkConn zkhelper.Conn, productName string) ([]*ServerGroup, error) {
	var ret []*ServerGroup
	root := fmt.Sprintf("/zk/codis/db_%s/servers", productName)
	groups, _, err := zkConn.Children(root)
	if err != nil {
		return nil, errors.Trace(err)
	}

	// Buggy :X
	//zkhelper.ChildrenRecursive(*zkConn, root)

	for _, group := range groups {
		// parse group_1 => 1
		groupId, err := strconv.Atoi(strings.Split(group, "_")[1])
		if err != nil {
			return nil, errors.Trace(err)
		}
		g, err := GetGroup(zkConn, productName, groupId)
		if err != nil {
			return nil, errors.Trace(err)
		}
		ret = append(ret, g)
	}
	return ret, nil
}

func (self *ServerGroup) Master(zkConn zkhelper.Conn) (*Server, error) {
	servers, err := self.GetServers(zkConn)
	if err != nil {
		return nil, errors.Trace(err)
	}
	for _, s := range servers {
		// TODO check if there are two masters
		if s.Type == SERVER_TYPE_MASTER {
			return s, nil
		}
	}
	return nil, nil
}

func (self *ServerGroup) Remove(zkConn zkhelper.Conn) error {
	// check if this group is not used by any slot
	slots, err := Slots(zkConn, self.ProductName)
	if err != nil {
		return errors.Trace(err)
	}

	for _, slot := range slots {
		if slot.GroupId == self.Id {
			return errors.Errorf("group %d is using by slot %d", slot.GroupId, slot.Id)
		}
	}

	// do delete
	zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d", self.ProductName, self.Id)
	err = zkhelper.DeleteRecursive(zkConn, zkPath, -1)

	// we know that there's no slots affected, so this action doesn't need proxy confirm
	err = NewAction(zkConn, self.ProductName, ACTION_TYPE_SERVER_GROUP_REMOVE, self, "", false)
	return errors.Trace(err)
}

func (self *ServerGroup) RemoveServer(zkConn zkhelper.Conn, addr string) error {
	zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d/%s", self.ProductName, self.Id, addr)
	data, _, err := zkConn.Get(zkPath)
	if err != nil {
		return errors.Trace(err)
	}

	var s Server
	err = json.Unmarshal(data, &s)
	if err != nil {
		return errors.Trace(err)
	}
	log.Info(s)
	if s.Type == SERVER_TYPE_MASTER {
		return errors.Errorf("cannot remove master, use promote first")
	}

	err = zkConn.Delete(zkPath, -1)
	if err != nil {
		return errors.Trace(err)
	}

	// update server list
	for i := 0; i < len(self.Servers); i++ {
		if self.Servers[i].Addr == s.Addr {
			self.Servers = append(self.Servers[:i], self.Servers[i+1:]...)
			break
		}
	}

	// remove slave won't need proxy confirm
	err = NewAction(zkConn, self.ProductName, ACTION_TYPE_SERVER_GROUP_CHANGED, self, "", false)
	return errors.Trace(err)
}

func (self *ServerGroup) Promote(conn zkhelper.Conn, addr, passwd string) error {
	var s *Server
	exists := false
	for i := 0; i < len(self.Servers); i++ {
		if self.Servers[i].Addr == addr {
			s = self.Servers[i]
			exists = true
			break
		}
	}

	if !exists {
		return errors.Errorf("no such addr %s", addr)
	}

	err := utils.SlaveNoOne(s.Addr, passwd)
	if err != nil {
		return errors.Trace(err)
	}

	// set origin master offline
	master, err := self.Master(conn)
	if err != nil {
		return errors.Trace(err)
	}

	// old master may be nil
	if master != nil {
		master.Type = SERVER_TYPE_OFFLINE
		err = self.AddServer(conn, master, passwd)
		if err != nil {
			return errors.Trace(err)
		}
	}

	// promote new server to master
	s.Type = SERVER_TYPE_MASTER
	err = self.AddServer(conn, s, passwd)
	return errors.Trace(err)
}

func (self *ServerGroup) Create(zkConn zkhelper.Conn) error {
	if self.Id < 0 {
		return errors.Errorf("invalid server group id %d", self.Id)
	}
	zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d", self.ProductName, self.Id)
	_, err := zkhelper.CreateOrUpdate(zkConn, zkPath, "", 0, zkhelper.DefaultDirACLs(), true)
	if err != nil {
		return errors.Trace(err)
	}
	err = NewAction(zkConn, self.ProductName, ACTION_TYPE_SERVER_GROUP_CHANGED, self, "", false)
	if err != nil {
		return errors.Trace(err)
	}

	// set no server slots' group id to this server group, no need to return error
	slots, err := NoGroupSlots(zkConn, self.ProductName)
	if err == nil && len(slots) > 0 {
		SetSlots(zkConn, self.ProductName, slots, self.Id, SLOT_STATUS_ONLINE)
	}

	return nil
}

func (self *ServerGroup) Exists(zkConn zkhelper.Conn) (bool, error) {
	zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d", self.ProductName, self.Id)
	b, err := zkhelper.NodeExists(zkConn, zkPath)
	if err != nil {
		return false, errors.Trace(err)
	}
	return b, nil
}

var ErrNodeExists = errors.New("node already exists")

func (self *ServerGroup) AddServer(zkConn zkhelper.Conn, s *Server, passwd string) error {
	s.GroupId = self.Id

	servers, err := self.GetServers(zkConn)
	if err != nil {
		return errors.Trace(err)
	}
	var masterAddr string
	for _, server := range servers {
		if server.Type == SERVER_TYPE_MASTER {
			masterAddr = server.Addr
		}
	}

	// make sure there is only one master
	if s.Type == SERVER_TYPE_MASTER && len(masterAddr) > 0 {
		return errors.Trace(ErrNodeExists)
	}

	// if this group has no server. auto promote this server to master
	if len(servers) == 0 {
		s.Type = SERVER_TYPE_MASTER
	}

	val, err := json.Marshal(s)
	if err != nil {
		return errors.Trace(err)
	}

	zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d/%s", self.ProductName, self.Id, s.Addr)
	_, err = zkhelper.CreateOrUpdate(zkConn, zkPath, string(val), 0, zkhelper.DefaultFileACLs(), true)

	// update servers
	servers, err = self.GetServers(zkConn)
	if err != nil {
		return errors.Trace(err)
	}
	self.Servers = servers

	if s.Type == SERVER_TYPE_MASTER {
		err = NewAction(zkConn, self.ProductName, ACTION_TYPE_SERVER_GROUP_CHANGED, self, "", true)
		if err != nil {
			return errors.Trace(err)
		}
	} else if s.Type == SERVER_TYPE_SLAVE && len(masterAddr) > 0 {
		// send command slaveof to slave
		err := utils.SlaveOf(s.Addr, passwd, masterAddr)
		if err != nil {
			return errors.Trace(err)
		}
	}

	return nil
}

func (self *ServerGroup) GetServers(zkConn zkhelper.Conn) ([]*Server, error) {
	var ret []*Server
	root := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d", self.ProductName, self.Id)
	nodes, _, err := zkConn.Children(root)
	if err != nil {
		return nil, errors.Trace(err)
	}
	for _, node := range nodes {
		nodePath := root + "/" + node
		s, err := GetServer(zkConn, nodePath)
		if err != nil {
			return nil, errors.Trace(err)
		}
		ret = append(ret, s)
	}
	return ret, nil
}
<commit_msg>check migerating slot when remove server group<commit_after>// Copyright 2014 Wandoujia Inc. All Rights Reserved.
// Licensed under the MIT (MIT-LICENSE.txt) license.

package models

import (
	"encoding/json"
	"fmt"
	"strconv"
	"strings"

	"github.com/wandoulabs/codis/pkg/utils"
	"github.com/wandoulabs/codis/pkg/utils/errors"
	"github.com/wandoulabs/codis/pkg/utils/log"
	"github.com/wandoulabs/zkhelper"
)

const (
	SERVER_TYPE_MASTER  string = "master"
	SERVER_TYPE_SLAVE   string = "slave"
	SERVER_TYPE_OFFLINE string = "offline"
)

// redis server instance
type Server struct {
	Type    string `json:"type"`
	GroupId int    `json:"group_id"`
	Addr    string `json:"addr"`
}

// redis server group
type ServerGroup struct {
	Id          int       `json:"id"`
	ProductName string    `json:"product_name"`
	Servers     []*Server `json:"servers"`
}

func (self *Server) String() string {
	b, _ := json.MarshalIndent(self, "", "  ")
	return string(b)
}

func (self *ServerGroup) String() string {
	b, _ := json.MarshalIndent(self, "", "  ")
	return string(b) + "\n"
}

func GetServer(zkConn zkhelper.Conn, zkPath string) (*Server, error) {
	data, _, err := zkConn.Get(zkPath)
	if err != nil {
		return nil, errors.Trace(err)
	}
	srv := Server{}
	if err := json.Unmarshal(data, &srv); err != nil {
		return nil, errors.Trace(err)
	}
	return &srv, nil
}

func NewServer(serverType string, addr string) *Server {
	return &Server{
		Type:    serverType,
		GroupId: INVALID_ID,
		Addr:    addr,
	}
}

func NewServerGroup(productName string, id int) *ServerGroup {
	return &ServerGroup{
		Id:          id,
		ProductName: productName,
	}
}

func GroupExists(zkConn zkhelper.Conn, productName string, groupId int) (bool, error) {
	zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d", productName, groupId)
	exists, _, err := zkConn.Exists(zkPath)
	if err != nil {
		return false, errors.Trace(err)
	}
	return exists, nil
}

func GetGroup(zkConn zkhelper.Conn, productName string, groupId int) (*ServerGroup, error) {
	exists, err := GroupExists(zkConn, productName, groupId)
	if err != nil {
		return nil, errors.Trace(err)
	}
	if !exists {
		return nil, errors.Errorf("group %d is not found", groupId)
	}

	group := &ServerGroup{
		ProductName: productName,
		Id:          groupId,
	}

	group.Servers, err = group.GetServers(zkConn)
	if err != nil {
		return nil, errors.Trace(err)
	}
	return group, nil
}

func ServerGroups(zkConn zkhelper.Conn, productName string) ([]*ServerGroup, error) {
	var ret []*ServerGroup
	root := fmt.Sprintf("/zk/codis/db_%s/servers", productName)
	groups, _, err := zkConn.Children(root)
	if err != nil {
		return nil, errors.Trace(err)
	}

	// Buggy :X
	//zkhelper.ChildrenRecursive(*zkConn, root)

	for _, group := range groups {
		// parse group_1 => 1
		groupId, err := strconv.Atoi(strings.Split(group, "_")[1])
		if err != nil {
			return nil, errors.Trace(err)
		}
		g, err := GetGroup(zkConn, productName, groupId)
		if err != nil {
			return nil, errors.Trace(err)
		}
		ret = append(ret, g)
	}
	return ret, nil
}

func (self *ServerGroup) Master(zkConn zkhelper.Conn) (*Server, error) {
	servers, err := self.GetServers(zkConn)
	if err != nil {
		return nil, errors.Trace(err)
	}
	for _, s := range servers {
		// TODO check if there are two masters
		if s.Type == SERVER_TYPE_MASTER {
			return s, nil
		}
	}
	return nil, nil
}

func (self *ServerGroup) Remove(zkConn zkhelper.Conn) error {
	// check if this group is not used by any slot
	slots, err := Slots(zkConn, self.ProductName)
	if err != nil {
		return errors.Trace(err)
	}

	for _, slot := range slots {
		if slot.GroupId == self.Id {
			return errors.Errorf("group %d is using by slot %d", slot.GroupId, slot.Id)
		}
		if ((slot.State.Status == SLOT_STATUS_MIGRATE || slot.State.Status == SLOT_STATUS_PRE_MIGRATE) && slot.State.MigrateStatus.From == self.Id) {
			return errors.Errorf("slot %d has residual data remain in group %d", slot.Id, self.Id)
		}
	}

	// do delete
	zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d", self.ProductName, self.Id)
	err = zkhelper.DeleteRecursive(zkConn, zkPath, -1)

	// we know that there's no slots affected, so this action doesn't need proxy confirm
	err = NewAction(zkConn, self.ProductName, ACTION_TYPE_SERVER_GROUP_REMOVE, self, "", false)
	return errors.Trace(err)
}

func (self *ServerGroup) RemoveServer(zkConn zkhelper.Conn, addr string) error {
	zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d/%s", self.ProductName, self.Id, addr)
	data, _, err := zkConn.Get(zkPath)
	if err != nil {
		return errors.Trace(err)
	}

	var s Server
	err = json.Unmarshal(data, &s)
	if err != nil {
		return errors.Trace(err)
	}
	log.Info(s)
	if s.Type == SERVER_TYPE_MASTER {
		return errors.Errorf("cannot remove master, use promote first")
	}

	err = zkConn.Delete(zkPath, -1)
	if err != nil {
		return errors.Trace(err)
	}

	// update server list
	for i := 0; i < len(self.Servers); i++ {
		if self.Servers[i].Addr == s.Addr {
			self.Servers = append(self.Servers[:i], self.Servers[i+1:]...)
			break
		}
	}

	// remove slave won't need proxy confirm
	err = NewAction(zkConn, self.ProductName, ACTION_TYPE_SERVER_GROUP_CHANGED, self, "", false)
	return errors.Trace(err)
}

func (self *ServerGroup) Promote(conn zkhelper.Conn, addr, passwd string) error {
	var s *Server
	exists := false
	for i := 0; i < len(self.Servers); i++ {
		if self.Servers[i].Addr == addr {
			s = self.Servers[i]
			exists = true
			break
		}
	}

	if !exists {
		return errors.Errorf("no such addr %s", addr)
	}

	err := utils.SlaveNoOne(s.Addr, passwd)
	if err != nil {
		return errors.Trace(err)
	}

	// set origin master offline
	master, err := self.Master(conn)
	if err != nil {
		return errors.Trace(err)
	}

	// old master may be nil
	if master != nil {
		master.Type = SERVER_TYPE_OFFLINE
		err = self.AddServer(conn, master, passwd)
		if err != nil {
			return errors.Trace(err)
		}
	}

	// promote new server to master
	s.Type = SERVER_TYPE_MASTER
	err = self.AddServer(conn, s, passwd)
	return errors.Trace(err)
}

func (self *ServerGroup) Create(zkConn zkhelper.Conn) error {
	if self.Id < 0 {
		return errors.Errorf("invalid server group id %d", self.Id)
	}
	zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d", self.ProductName, self.Id)
	_, err := zkhelper.CreateOrUpdate(zkConn, zkPath, "", 0, zkhelper.DefaultDirACLs(), true)
	if err != nil {
		return errors.Trace(err)
	}
	err = NewAction(zkConn, self.ProductName, ACTION_TYPE_SERVER_GROUP_CHANGED, self, "", false)
	if err != nil {
		return errors.Trace(err)
	}

	// set no server slots' group id to this server group, no need to return error
	slots, err := NoGroupSlots(zkConn, self.ProductName)
	if err == nil && len(slots) > 0 {
		SetSlots(zkConn, self.ProductName, slots, self.Id, SLOT_STATUS_ONLINE)
	}

	return nil
}

func (self *ServerGroup) Exists(zkConn zkhelper.Conn) (bool, error) {
	zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d", self.ProductName, self.Id)
	b, err := zkhelper.NodeExists(zkConn, zkPath)
	if err != nil {
		return false, errors.Trace(err)
	}
	return b, nil
}

var ErrNodeExists = errors.New("node already exists")

func (self *ServerGroup) AddServer(zkConn zkhelper.Conn, s *Server, passwd string) error {
	s.GroupId = self.Id

	servers, err := self.GetServers(zkConn)
	if err != nil {
		return errors.Trace(err)
	}
	var masterAddr string
	for _, server := range servers {
		if server.Type == SERVER_TYPE_MASTER {
			masterAddr = server.Addr
		}
	}

	// make sure there is only one master
	if s.Type == SERVER_TYPE_MASTER && len(masterAddr) > 0 {
		return errors.Trace(ErrNodeExists)
	}

	// if this group has no server. auto promote this server to master
	if len(servers) == 0 {
		s.Type = SERVER_TYPE_MASTER
	}

	val, err := json.Marshal(s)
	if err != nil {
		return errors.Trace(err)
	}

	zkPath := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d/%s", self.ProductName, self.Id, s.Addr)
	_, err = zkhelper.CreateOrUpdate(zkConn, zkPath, string(val), 0, zkhelper.DefaultFileACLs(), true)

	// update servers
	servers, err = self.GetServers(zkConn)
	if err != nil {
		return errors.Trace(err)
	}
	self.Servers = servers

	if s.Type == SERVER_TYPE_MASTER {
		err = NewAction(zkConn, self.ProductName, ACTION_TYPE_SERVER_GROUP_CHANGED, self, "", true)
		if err != nil {
			return errors.Trace(err)
		}
	} else if s.Type == SERVER_TYPE_SLAVE && len(masterAddr) > 0 {
		// send command slaveof to slave
		err := utils.SlaveOf(s.Addr, passwd, masterAddr)
		if err != nil {
			return errors.Trace(err)
		}
	}

	return nil
}

func (self *ServerGroup) GetServers(zkConn zkhelper.Conn) ([]*Server, error) {
	var ret []*Server
	root := fmt.Sprintf("/zk/codis/db_%s/servers/group_%d", self.ProductName, self.Id)
	nodes, _, err := zkConn.Children(root)
	if err != nil {
		return nil, errors.Trace(err)
	}
	for _, node := range nodes {
		nodePath := root + "/" + node
		s, err := GetServer(zkConn, nodePath)
		if err != nil {
			return nil, errors.Trace(err)
		}
		ret = append(ret, s)
	}
	return ret, nil
}
<|endoftext|>


<commit_before>package sharings

import (
	"fmt"
	"net/url"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/cozy/cozy-stack/pkg/stack"
	"github.com/cozy/cozy-stack/pkg/workers/mails"
)

// The sharing-dependant information: the recipient's name, the sharer's public
// name, the description of the sharing, and the OAuth query string.
type mailTemplateValues struct {
	RecipientName    string
	SharerPublicName string
	Description      string
	SharingLink      string
}

// SendDiscoveryMail send a mail to the recipient, in order for him to give his
// URL to the sender
func SendDiscoveryMail(instance *instance.Instance, s *Sharing, rs *RecipientStatus) error {
	sharerPublicName, err := instance.PublicName()
	if err != nil {
		return err
	}
	// Fill in the description.
	desc := s.Desc
	if desc == "" {
		desc = "[No description provided]"
	}
	discoveryLink, err := generateDiscoveryLink(instance, s, rs)
	if err != nil {
		return err
	}

	// Generate the base values of the email to send
	discoveryMsg, err := generateMailMessage(s, rs.recipient,
		&mailTemplateValues{
			RecipientName:    rs.recipient.Email,
			SharerPublicName: sharerPublicName,
			Description:      desc,
			SharingLink:      discoveryLink,
		},
	)
	if err != nil {
		return err
	}
	_, err = stack.GetBroker().PushJob(&jobs.JobRequest{
		Domain:     instance.Domain,
		WorkerType: "sendmail",
		Options:    nil,
		Message:    discoveryMsg,
	})
	return err
}

// SendSharingMails will generate the mail containing the details
// regarding this sharing, and will then send it to all the recipients.
func SendSharingMails(instance *instance.Instance, s *Sharing) error {
	sharerPublicName, err := instance.PublicName()
	if err != nil {
		return err
	}
	// Fill in the description.
	desc := s.Desc
	if desc == "" {
		desc = "[No description provided]"
	}

	errorOccurred := false
	for _, rs := range s.RecipientsStatus {
		err = rs.GetRecipient(instance)
		if err != nil {
			return err
		}
		if rs.recipient.Email == "" {
			errorOccurred = logError(instance, ErrRecipientHasNoEmail)
			continue
		}
		// Special case if the recipient's URL is not known: start discovery
		if rs.recipient.URL == "" {
			return SendDiscoveryMail(instance, s, rs)
		}
		// Send mail based on the recipient status
		if rs.Status == consts.SharingStatusMailNotSent {
			// Generate recipient specific OAuth query string.
			oAuthStr, errOAuth := GenerateOAuthQueryString(s, rs, instance.Scheme())
			if errOAuth != nil {
				errorOccurred = logError(instance, errOAuth)
				continue
			}

			// Generate the base values of the email to send, common to all
			// recipients: the description and the sharer's public name.
			sharingMessage, errGenMail := generateMailMessage(s, rs.recipient,
				&mailTemplateValues{
					RecipientName:    rs.recipient.Email,
					SharerPublicName: sharerPublicName,
					Description:      desc,
					SharingLink:      oAuthStr,
				},
			)
			if errGenMail != nil {
				errorOccurred = logError(instance, errGenMail)
				continue
			}

			// We ask to queue a new mail job.
			// The returned values (other than the error) are ignored because they
			// are of no use in this situation.
			// FI: they correspond to the job information and to a channel with
			// which we can check the advancement of said job.
			_, errJobs := stack.GetBroker().PushJob(&jobs.JobRequest{
				Domain:     instance.Domain,
				WorkerType: "sendmail",
				Options:    nil,
				Message:    sharingMessage,
			})
			if errJobs != nil {
				errorOccurred = logError(instance, errJobs)
				continue
			}

			// Job was created, we set the status to "pending".
			rs.Status = consts.SharingStatusPending
		}
	}
	// Persist the modifications in the database.
	err = couchdb.UpdateDoc(instance, s)
	if err != nil {
		if errorOccurred {
			return fmt.Errorf("[sharing] Error updating the document (%v) "+
				"and sending the email invitation (%v)", err,
				ErrMailCouldNotBeSent)
		}

		return err
	}

	if errorOccurred {
		return ErrMailCouldNotBeSent
	}

	return nil
}

// logError will log an error in the stack.
func logError(i *instance.Instance, err error) bool {
	i.Logger().Error("[sharing] An error occurred while trying to send the email "+
		"invitation: ", err)
	return true
}

// generateMailMessage will extract and compute the relevant information
// from the sharing to generate the mail we will send to the specified
// recipient.
func generateMailMessage(s *Sharing, r *Recipient, mailValues *mailTemplateValues) (*jobs.Message, error) {
	if r.Email == "" {
		return nil, ErrRecipientHasNoEmail
	}
	mailAddresses := []*mails.Address{&mails.Address{
		Name:  r.Email,
		Email: r.Email,
	}}
	return jobs.NewMessage(jobs.JSONEncoding, mails.Options{
		Mode:           "from",
		To:             mailAddresses,
		Subject:        "New sharing request / Nouvelle demande de partage",
		TemplateName:   "sharing_request",
		TemplateValues: mailValues,
	})
}

// GenerateOAuthQueryString takes care of creating a correct OAuth request for
// the given sharing and recipient.
func GenerateOAuthQueryString(s *Sharing, rs *RecipientStatus, scheme string) (string, error) {

	// Check if an oauth client exists for the owner at the recipient's.
	if rs.Client.ClientID == "" || len(rs.Client.RedirectURIs) < 1 {
		return "", ErrNoOAuthClient
	}

	// Check if the recipient has an URL.
	if rs.recipient.URL == "" {
		return "", ErrRecipientHasNoURL
	}

	// In the sharing document the permissions are stored as a
	// `permissions.Set`. We need to convert them in a proper format to be able
	// to incorporate them in the OAuth query string.
	//
	// Optimization: the next four lines of code could be outside of this
	// function and also outside of the for loop that iterates on the
	// recipients in `SendSharingMails`.
	// I found it was clearer to leave it here, at the price of being less
	// optimized.
	permissionsScope, err := s.Permissions.MarshalScopeString()
	if err != nil {
		return "", err
	}

	oAuthQuery, err := url.Parse(rs.recipient.URL)
	if err != nil {
		return "", err
	}
	// Special scenario: if r.URL doesn't have an "http://" or "https://" prefix
	// then `url.Parse` doesn't set any host.
	if oAuthQuery.Host == "" {
		oAuthQuery.Host = rs.recipient.URL
	}
	oAuthQuery.Path = "/sharings/request"
	// The link/button we put in the email has to have an http:// or https://
	// prefix, otherwise it cannot be open in the browser.
	if oAuthQuery.Scheme != "http" && oAuthQuery.Scheme != "https" {
		oAuthQuery.Scheme = scheme
	}

	mapParamOAuthQuery := url.Values{
		consts.QueryParamAppSlug: {s.AppSlug},
		"client_id":              {rs.Client.ClientID},
		"redirect_uri":           {rs.Client.RedirectURIs[0]},
		"response_type":          {"code"},
		"scope":                  {permissionsScope},
		"sharing_type":           {s.SharingType},
		"state":                  {s.SharingID},
	}
	oAuthQuery.RawQuery = mapParamOAuthQuery.Encode()

	return oAuthQuery.String(), nil
}

func generateDiscoveryLink(instance *instance.Instance, s *Sharing, rs *RecipientStatus) (string, error) {
	// Check if the recipient has an URL.
	if rs.recipient.Email == "" {
		return "", ErrRecipientHasNoEmail
	}

	path := "/sharings/discovery"
	discQuery := url.Values{
		"recipient_id":    {rs.recipient.ID()},
		"sharing_id":      {s.SharingID},
		"recipient_email": {rs.recipient.Email},
	}
	discURL := url.URL{
		Scheme:   instance.Scheme(),
		Host:     instance.Domain,
		Path:     path,
		RawQuery: discQuery.Encode(),
	}

	return discURL.String(), nil
}
<commit_msg>Save recipient status for discovery<commit_after>package sharings

import (
	"fmt"
	"net/url"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/cozy/cozy-stack/pkg/stack"
	"github.com/cozy/cozy-stack/pkg/workers/mails"
)

// The sharing-dependant information: the recipient's name, the sharer's public
// name, the description of the sharing, and the OAuth query string.
type mailTemplateValues struct {
	RecipientName    string
	SharerPublicName string
	Description      string
	SharingLink      string
}

// SendDiscoveryMail send a mail to the recipient, in order for him to give his
// URL to the sender
func SendDiscoveryMail(instance *instance.Instance, s *Sharing, rs *RecipientStatus) error {
	sharerPublicName, err := instance.PublicName()
	if err != nil {
		return err
	}
	// Fill in the description.
	desc := s.Desc
	if desc == "" {
		desc = "[No description provided]"
	}
	discoveryLink, err := generateDiscoveryLink(instance, s, rs)
	if err != nil {
		return err
	}

	// Generate the base values of the email to send
	discoveryMsg, err := generateMailMessage(s, rs.recipient,
		&mailTemplateValues{
			RecipientName:    rs.recipient.Email,
			SharerPublicName: sharerPublicName,
			Description:      desc,
			SharingLink:      discoveryLink,
		},
	)
	if err != nil {
		return err
	}
	_, err = stack.GetBroker().PushJob(&jobs.JobRequest{
		Domain:     instance.Domain,
		WorkerType: "sendmail",
		Options:    nil,
		Message:    discoveryMsg,
	})
	return err
}

// SendSharingMails will generate the mail containing the details
// regarding this sharing, and will then send it to all the recipients.
func SendSharingMails(instance *instance.Instance, s *Sharing) error {
	sharerPublicName, err := instance.PublicName()
	if err != nil {
		return err
	}
	// Fill in the description.
	desc := s.Desc
	if desc == "" {
		desc = "[No description provided]"
	}

	errorOccurred := false
	for _, rs := range s.RecipientsStatus {
		err = rs.GetRecipient(instance)
		if err != nil {
			return err
		}
		if rs.recipient.Email == "" {
			errorOccurred = logError(instance, ErrRecipientHasNoEmail)
			continue
		}
		// Special case if the recipient's URL is not known: start discovery
		if rs.recipient.URL == "" {
			err = SendDiscoveryMail(instance, s, rs)
			if err != nil {
				logError(instance, err)
				rs.Status = consts.SharingStatusMailNotSent
			} else {
				rs.Status = consts.SharingStatusPending
			}
			return couchdb.UpdateDoc(instance, s)
		}
		// Send mail based on the recipient status
		if rs.Status == consts.SharingStatusMailNotSent {
			// Generate recipient specific OAuth query string.
			oAuthStr, errOAuth := GenerateOAuthQueryString(s, rs, instance.Scheme())
			if errOAuth != nil {
				errorOccurred = logError(instance, errOAuth)
				continue
			}

			// Generate the base values of the email to send, common to all
			// recipients: the description and the sharer's public name.
			sharingMessage, errGenMail := generateMailMessage(s, rs.recipient,
				&mailTemplateValues{
					RecipientName:    rs.recipient.Email,
					SharerPublicName: sharerPublicName,
					Description:      desc,
					SharingLink:      oAuthStr,
				},
			)
			if errGenMail != nil {
				errorOccurred = logError(instance, errGenMail)
				continue
			}

			// We ask to queue a new mail job.
			// The returned values (other than the error) are ignored because they
			// are of no use in this situation.
			// FI: they correspond to the job information and to a channel with
			// which we can check the advancement of said job.
			_, errJobs := stack.GetBroker().PushJob(&jobs.JobRequest{
				Domain:     instance.Domain,
				WorkerType: "sendmail",
				Options:    nil,
				Message:    sharingMessage,
			})
			if errJobs != nil {
				errorOccurred = logError(instance, errJobs)
				continue
			}

			// Job was created, we set the status to "pending".
			rs.Status = consts.SharingStatusPending
		}
	}
	// Persist the modifications in the database.
	err = couchdb.UpdateDoc(instance, s)
	if err != nil {
		if errorOccurred {
			return fmt.Errorf("[sharing] Error updating the document (%v) "+
				"and sending the email invitation (%v)", err,
				ErrMailCouldNotBeSent)
		}
		return err
	}

	if errorOccurred {
		return ErrMailCouldNotBeSent
	}

	return nil
}

// logError will log an error in the stack.
func logError(i *instance.Instance, err error) bool {
	i.Logger().Error("[sharing] An error occurred while trying to send the email "+
		"invitation: ", err)
	return true
}

// generateMailMessage will extract and compute the relevant information
// from the sharing to generate the mail we will send to the specified
// recipient.
func generateMailMessage(s *Sharing, r *Recipient, mailValues *mailTemplateValues) (*jobs.Message, error) {
	if r.Email == "" {
		return nil, ErrRecipientHasNoEmail
	}
	mailAddresses := []*mails.Address{&mails.Address{
		Name:  r.Email,
		Email: r.Email,
	}}
	return jobs.NewMessage(jobs.JSONEncoding, mails.Options{
		Mode:           "from",
		To:             mailAddresses,
		Subject:        "New sharing request / Nouvelle demande de partage",
		TemplateName:   "sharing_request",
		TemplateValues: mailValues,
	})
}

// GenerateOAuthQueryString takes care of creating a correct OAuth request for
// the given sharing and recipient.
func GenerateOAuthQueryString(s *Sharing, rs *RecipientStatus, scheme string) (string, error) {

	// Check if an oauth client exists for the owner at the recipient's.
	if rs.Client.ClientID == "" || len(rs.Client.RedirectURIs) < 1 {
		return "", ErrNoOAuthClient
	}

	// Check if the recipient has an URL.
	if rs.recipient.URL == "" {
		return "", ErrRecipientHasNoURL
	}

	// In the sharing document the permissions are stored as a
	// `permissions.Set`. We need to convert them in a proper format to be able
	// to incorporate them in the OAuth query string.
	//
	// Optimization: the next four lines of code could be outside of this
	// function and also outside of the for loop that iterates on the
	// recipients in `SendSharingMails`.
	// I found it was clearer to leave it here, at the price of being less
	// optimized.
	permissionsScope, err := s.Permissions.MarshalScopeString()
	if err != nil {
		return "", err
	}

	oAuthQuery, err := url.Parse(rs.recipient.URL)
	if err != nil {
		return "", err
	}
	// Special scenario: if r.URL doesn't have an "http://" or "https://" prefix
	// then `url.Parse` doesn't set any host.
	if oAuthQuery.Host == "" {
		oAuthQuery.Host = rs.recipient.URL
	}
	oAuthQuery.Path = "/sharings/request"
	// The link/button we put in the email has to have an http:// or https://
	// prefix, otherwise it cannot be open in the browser.
	if oAuthQuery.Scheme != "http" && oAuthQuery.Scheme != "https" {
		oAuthQuery.Scheme = scheme
	}

	mapParamOAuthQuery := url.Values{
		consts.QueryParamAppSlug: {s.AppSlug},
		"client_id":              {rs.Client.ClientID},
		"redirect_uri":           {rs.Client.RedirectURIs[0]},
		"response_type":          {"code"},
		"scope":                  {permissionsScope},
		"sharing_type":           {s.SharingType},
		"state":                  {s.SharingID},
	}
	oAuthQuery.RawQuery = mapParamOAuthQuery.Encode()

	return oAuthQuery.String(), nil
}

func generateDiscoveryLink(instance *instance.Instance, s *Sharing, rs *RecipientStatus) (string, error) {
	// Check if the recipient has an URL.
	if rs.recipient.Email == "" {
		return "", ErrRecipientHasNoEmail
	}

	path := "/sharings/discovery"
	discQuery := url.Values{
		"recipient_id":    {rs.recipient.ID()},
		"sharing_id":      {s.SharingID},
		"recipient_email": {rs.recipient.Email},
	}
	discURL := url.URL{
		Scheme:   instance.Scheme(),
		Host:     instance.Domain,
		Path:     path,
		RawQuery: discQuery.Encode(),
	}

	return discURL.String(), nil
}
<|endoftext|>


<commit_before>package snmpquery

import (
	"math/rand"
	"strconv"
	"time"
)

type OpSnmp int32

const (
	GET  = 0
	WALK = 1
)

type Query struct {
	Id          int
	Cmd         OpSnmp
	Community   string
	Oid         string
	Destination string
	Response    string
	Error       int
}

func NewQuery(id int, cmd OpSnmp, destination, community, oid string) *Query {
	return &Query{
		Id:          id,
		Cmd:         cmd,
		Community:   community,
		Oid:         oid,
		Destination: destination,
	}
}

func Process(input chan Query, processed chan Query, conntention int) {
	m := make(map[string]chan Query)

	for query := range input {
		_, exists := m[query.Destination]
		if exists == false {
			channel_tmp := make(chan Query, 10)
			m[query.Destination] = channel_tmp
			for i := 0; i < conntention; i++ {
				go processQueriesFromChannel(channel_tmp, processed)
			}
		}
		m[query.Destination] <- query
	}
}

func handleQuery(query *Query) {
	time.Sleep(time.Duration(rand.Intn(1e3)) * time.Millisecond)
	query.Response = "whatever " + strconv.Itoa(rand.Intn(1e3))
}

func processQueriesFromChannel(input chan Query, processed chan Query) {
	for query := range input {
		handleQuery(&query)
		processed <- query
	}
}
<commit_msg>First tests executing a walk...<commit_after>package snmpquery

import (
	"fmt"
	"math/rand"
	"strconv"
	"time"

	"github.com/soniah/gosnmp"
)

type OpSnmp int32

const (
	GET  = 0
	WALK = 1
)

type Query struct {
	Id          int
	Cmd         OpSnmp
	Community   string
	Oid         string
	Destination string
	Response    string
	Error       int
}

func NewQuery(id int, cmd OpSnmp, destination, community, oid string) *Query {
	return &Query{
		Id:          id,
		Cmd:         cmd,
		Community:   community,
		Oid:         oid,
		Destination: destination,
	}
}

func Process(input chan Query, processed chan Query, conntention int) {
	m := make(map[string]chan Query)

	for query := range input {
		_, exists := m[query.Destination]
		if exists == false {
			channel_tmp := make(chan Query, 10)
			m[query.Destination] = channel_tmp
			for i := 0; i < conntention; i++ {
				go processQueriesFromChannel(channel_tmp, processed)
			}
		}
		m[query.Destination] <- query
	}
}

func printValue(pdu gosnmp.SnmpPDU) error {
	fmt.Printf("%s = ", pdu.Name)

	switch pdu.Type {
	case gosnmp.OctetString:
		fmt.Printf("STRING: %s\n", pdu.Value.(string))
	case gosnmp.Counter64:
		fmt.Printf("COUNTER64 %d: %d\n", pdu.Type, gosnmp.ToBigInt(pdu.Value))
	default:
		fmt.Printf("TYPE %d: %d\n", pdu.Type, gosnmp.ToBigInt(pdu.Value))
	}
	return nil
}

func handleQuery(query *Query) {

	gosnmp.Default.Community = query.Community
	gosnmp.Default.Target = query.Destination
	gosnmp.Default.Timeout = time.Duration(10 * time.Second)
	err := gosnmp.Default.Connect()
	if err != nil {
		fmt.Printf("Connect err: %v\n", err)
		return
	}
	defer gosnmp.Default.Conn.Close()

	err = gosnmp.Default.BulkWalk(query.Oid, printValue)
	if err != nil {
		fmt.Printf("Walk Error: %v\n", err)
		return
	}

	query.Response = "whatever " + strconv.Itoa(rand.Intn(1e3))
}

func processQueriesFromChannel(input chan Query, processed chan Query) {
	for query := range input {
		handleQuery(&query)
		processed <- query
	}
}
<|endoftext|>