<commit_before>package cli

import (
	"bufio"
	"compress/gzip"
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"
	oauthv1 "github.com/openshift/api/oauth/v1"
	"github.com/openshift/client-go/image/clientset/versioned"
	oauthv1client "github.com/openshift/client-go/oauth/clientset/versioned/typed/oauth/v1"
	"github.com/openshift/library-go/pkg/image/imageutil"
	exutil "github.com/openshift/origin/test/extended/util"
	"github.com/openshift/origin/test/extended/util/ibmcloud"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	e2e "k8s.io/kubernetes/test/e2e/framework"
)

var _ = g.Describe("[sig-cli] oc adm must-gather", func() {
	defer g.GinkgoRecover()
	oc := exutil.NewCLI("oc-adm-must-gather").AsAdmin()

	g.JustBeforeEach(func() {
		// wait for the default service account to be avaiable
		err := exutil.WaitForServiceAccount(oc.KubeClient().CoreV1().ServiceAccounts(oc.Namespace()), "default")
		o.Expect(err).NotTo(o.HaveOccurred())
	})

	g.It("runs successfully", func() {
		tempDir, err := ioutil.TempDir("", "test.oc-adm-must-gather.")
		o.Expect(err).NotTo(o.HaveOccurred())
		defer os.RemoveAll(tempDir)
		o.Expect(oc.Run("adm", "must-gather").Args("--dest-dir", tempDir).Execute()).To(o.Succeed())

		pluginOutputDir := getPluginOutputDir(oc, tempDir)

		expectedDirectories := [][]string{
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io"},
			{pluginOutputDir, "cluster-scoped-resources", "operator.openshift.io"},
			{pluginOutputDir, "cluster-scoped-resources", "core"},
			{pluginOutputDir, "cluster-scoped-resources", "apiregistration.k8s.io"},
			{pluginOutputDir, "namespaces", "openshift"},
			{pluginOutputDir, "namespaces", "openshift-kube-apiserver-operator"},
		}

		expectedFiles := [][]string{
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "apiservers.yaml"},
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "authentications.yaml"},
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "builds.yaml"},
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "clusteroperators.yaml"},
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "clusterversions.yaml"},
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "consoles.yaml"},
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "dnses.yaml"},
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "featuregates.yaml"},
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "images.yaml"},
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "infrastructures.yaml"},
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "ingresses.yaml"},
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "networks.yaml"},
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "oauths.yaml"},
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "projects.yaml"},
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "schedulers.yaml"},
			// TODO: This got broken and we need to fix this. Disabled temporarily.
			// {pluginOutputDir, "namespaces", "openshift-kube-apiserver", "core", "configmaps.yaml"},
			// {pluginOutputDir, "namespaces", "openshift-kube-apiserver", "core", "secrets.yaml"},
			{pluginOutputDir, "host_service_logs", "masters", "crio_service.log"},
			{pluginOutputDir, "host_service_logs", "masters", "kubelet_service.log"},
		}

		for _, expectedDirectory := range expectedDirectories {
			o.Expect(path.Join(expectedDirectory...)).To(o.BeADirectory())
		}

		emptyFiles := []string{}
		for _, expectedFile := range expectedFiles {
			expectedFilePath := path.Join(expectedFile...)
			o.Expect(expectedFilePath).To(o.BeAnExistingFile())
			stat, err := os.Stat(expectedFilePath)
			o.Expect(err).NotTo(o.HaveOccurred())
			if size := stat.Size(); size < 50 {
				emptyFiles = append(emptyFiles, expectedFilePath)
			}
		}
		if len(emptyFiles) > 0 {
			o.Expect(fmt.Errorf("expected files should not be empty: %s", strings.Join(emptyFiles, ","))).NotTo(o.HaveOccurred())
		}
	})

	g.It("runs successfully with options", func() {
		tempDir, err := ioutil.TempDir("", "test.oc-adm-must-gather.")
		o.Expect(err).NotTo(o.HaveOccurred())
		defer os.RemoveAll(tempDir)
		args := []string{
			"--dest-dir", tempDir,
			"--source-dir", "/artifacts",
			"--",
			"/bin/bash", "-c",
			"ls -l > /artifacts/ls.log",
		}
		o.Expect(oc.Run("adm", "must-gather").Args(args...).Execute()).To(o.Succeed())
		expectedFilePath := path.Join(getPluginOutputDir(oc, tempDir), "ls.log")
		o.Expect(expectedFilePath).To(o.BeAnExistingFile())
		stat, err := os.Stat(expectedFilePath)
		o.Expect(err).NotTo(o.HaveOccurred())
		o.Expect(stat.Size()).To(o.BeNumerically(">", 0))
	})

	g.It("runs successfully for audit logs", func() {
		g.Skip("this test is flaking like crazy on Azure (not sure of other platforms")

		// On IBM ROKS, events will not be part of the output, since audit logs do not include control plane logs.
		if e2e.TestContext.Provider == ibmcloud.ProviderName {
			g.Skip("ROKs doesn't have audit logs")
		}

		// makes some tokens that should not show in the audit logs
		const tokenName = "must-gather-audit-logs-token-plus-some-padding-here-to-make-the-limit"
		oauthClient := oauthv1client.NewForConfigOrDie(oc.AdminConfig())
		_, err1 := oauthClient.OAuthAccessTokens().Create(context.Background(), &oauthv1.OAuthAccessToken{
			ObjectMeta: metav1.ObjectMeta{
				Name: tokenName,
			},
			ClientName:  "openshift-challenging-client",
			ExpiresIn:   30,
			Scopes:      []string{"user:info"},
			RedirectURI: "https://127.0.0.1:12000/oauth/token/implicit",
			UserName:    "a",
			UserUID:     "1",
		}, metav1.CreateOptions{})
		o.Expect(err1).NotTo(o.HaveOccurred())
		_, err2 := oauthClient.OAuthAuthorizeTokens().Create(context.Background(), &oauthv1.OAuthAuthorizeToken{
			ObjectMeta: metav1.ObjectMeta{
				Name: tokenName,
			},
			ClientName:  "openshift-challenging-client",
			ExpiresIn:   30,
			Scopes:      []string{"user:info"},
			RedirectURI: "https://127.0.0.1:12000/oauth/token/implicit",
			UserName:    "a",
			UserUID:     "1",
		}, metav1.CreateOptions{})
		o.Expect(err2).NotTo(o.HaveOccurred())

		tempDir, err := ioutil.TempDir("", "test.oc-adm-must-gather.")
		o.Expect(err).NotTo(o.HaveOccurred())
		// defer os.RemoveAll(tempDir)

		args := []string{
			"--dest-dir", tempDir,
			"--",
			"/usr/bin/gather_audit_logs",
		}

		o.Expect(oc.Run("adm", "must-gather").Args(args...).Execute()).To(o.Succeed())
		// wait for the contents to show up in the plugin output directory, avoiding EOF errors
		time.Sleep(10 * time.Second)

		pluginOutputDir := getPluginOutputDir(oc, tempDir)

		expectedDirectoriesToExpectedCount := map[string]int{
			path.Join(pluginOutputDir, "audit_logs", "kube-apiserver"):      1000,
			path.Join(pluginOutputDir, "audit_logs", "openshift-apiserver"): 10, // openshift apiservers don't necessarily get much traffic.  Especially early in a run
		}

		expectedFiles := [][]string{
			{pluginOutputDir, "audit_logs", "kube-apiserver.audit_logs_listing"},
			{pluginOutputDir, "audit_logs", "openshift-apiserver.audit_logs_listing"},
		}

		// for some crazy reason, it seems that the files from must-gather take time to appear on disk for reading.  I don't understand why
		// but this was in a previous commit and I don't want to immediately flake: https://github.com/openshift/origin/commit/006745a535848e84dcbcdd1c83ae86deddd3a229#diff-ad1c47fa4213de16d8b3237df5d71724R168
		// so we're going to try to get a pass every 10 seconds for a minute.  If we pass, great.  If we don't, we report the
		// last error we had.
		var lastErr error
		err = wait.PollImmediate(10*time.Second, 1*time.Minute, func() (bool, error) {
			// make sure we do not log OAuth tokens
			for auditDirectory, expectedNumberOfAuditEntries := range expectedDirectoriesToExpectedCount {
				eventsChecked := 0
				err := filepath.Walk(auditDirectory, func(path string, info os.FileInfo, err error) error {
					g.By(path)
					o.Expect(err).NotTo(o.HaveOccurred())
					if info.IsDir() {
						return nil
					}

					if (strings.Contains(path, "-termination-") && strings.HasSuffix(path, ".log.gz")) || strings.HasSuffix(path, "termination.log.gz") {
						// these are expected, but have unstructured log format
						return nil
					}

					isAuditFile := (strings.Contains(path, "-audit-") && strings.HasSuffix(path, ".log.gz")) || strings.HasSuffix(path, "audit.log.gz")
					o.Expect(isAuditFile).To(o.BeTrue())

					// at this point, we expect only audit files with json events, one per line

					readFile := false

					file, err := os.Open(path)
					o.Expect(err).NotTo(o.HaveOccurred())
					defer file.Close()

					fi, err := file.Stat()
					o.Expect(err).NotTo(o.HaveOccurred())

					// it will happen that the audit files are sometimes empty, we can
					// safely ignore these files since they don't provide valuable information
					// TODO this doesn't seem right.  It should be really unlikely, but we'll deal with later
					if fi.Size() == 0 {
						return nil
					}

					gzipReader, err := gzip.NewReader(file)
					o.Expect(err).NotTo(o.HaveOccurred())

					scanner := bufio.NewScanner(gzipReader)
					for scanner.Scan() {
						text := scanner.Text()
						if !strings.HasSuffix(text, "}") {
							continue // ignore truncated data
						}
						o.Expect(text).To(o.HavePrefix(`{"kind":"Event",`))
						for _, token := range []string{"oauthaccesstokens", "oauthauthorizetokens", tokenName} {
							o.Expect(text).NotTo(o.ContainSubstring(token))
						}
						readFile = true
						eventsChecked++
					}
					// ignore this error as we usually fail to read the whole GZ file
					// o.Expect(scanner.Err()).NotTo(o.HaveOccurred())
					o.Expect(readFile).To(o.BeTrue())

					return nil
				})
				o.Expect(err).NotTo(o.HaveOccurred())

				if eventsChecked <= expectedNumberOfAuditEntries {
					lastErr = fmt.Errorf("expected %d audit events for %q, but only got %d", expectedNumberOfAuditEntries, auditDirectory, eventsChecked)
					return false, nil
				}

				// reset lastErr if we succeeded.
				lastErr = nil
			}

			// if we get here, it means both directories checked out ok
			return true, nil
		})
		o.Expect(lastErr).NotTo(o.HaveOccurred()) // print the last error first if we have one
		o.Expect(err).NotTo(o.HaveOccurred())     // otherwise be sure we fail on the timeout if it happened

		emptyFiles := []string{}
		for _, expectedFile := range expectedFiles {
			expectedFilePath := path.Join(expectedFile...)
			o.Expect(expectedFilePath).To(o.BeAnExistingFile())
			stat, err := os.Stat(expectedFilePath)
			o.Expect(err).NotTo(o.HaveOccurred())
			if size := stat.Size(); size < 50 {
				emptyFiles = append(emptyFiles, expectedFilePath)
			}
		}
		if len(emptyFiles) > 0 {
			o.Expect(fmt.Errorf("expected files should not be empty: %s", strings.Join(emptyFiles, ","))).NotTo(o.HaveOccurred())
		}
	})
})

func getPluginOutputDir(oc *exutil.CLI, tempDir string) string {
	imageClient := versioned.NewForConfigOrDie(oc.AdminConfig())
	stream, err := imageClient.ImageV1().ImageStreams("openshift").Get(context.Background(), "must-gather", metav1.GetOptions{})
	o.Expect(err).NotTo(o.HaveOccurred())
	imageId, ok := imageutil.ResolveLatestTaggedImage(stream, "latest")
	o.Expect(ok).To(o.BeTrue())
	pluginOutputDir := path.Join(tempDir, regexp.MustCompile("[^A-Za-z0-9]+").ReplaceAllString(imageId, "-"))
	fileInfo, err := os.Stat(pluginOutputDir)
	if err != nil || !fileInfo.IsDir() {
		pluginOutputDir = tempDir
	}
	return pluginOutputDir
}
<commit_msg>Bug 1861746: bring back must-gather audit e2e<commit_after>package cli

import (
	"bufio"
	"compress/gzip"
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"
	oauthv1 "github.com/openshift/api/oauth/v1"
	"github.com/openshift/client-go/image/clientset/versioned"
	oauthv1client "github.com/openshift/client-go/oauth/clientset/versioned/typed/oauth/v1"
	"github.com/openshift/library-go/pkg/image/imageutil"
	exutil "github.com/openshift/origin/test/extended/util"
	"github.com/openshift/origin/test/extended/util/ibmcloud"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	e2e "k8s.io/kubernetes/test/e2e/framework"
)

var _ = g.Describe("[sig-cli] oc adm must-gather", func() {
	defer g.GinkgoRecover()
	oc := exutil.NewCLI("oc-adm-must-gather").AsAdmin()

	g.JustBeforeEach(func() {
		// wait for the default service account to be avaiable
		err := exutil.WaitForServiceAccount(oc.KubeClient().CoreV1().ServiceAccounts(oc.Namespace()), "default")
		o.Expect(err).NotTo(o.HaveOccurred())
	})

	g.It("runs successfully", func() {
		tempDir, err := ioutil.TempDir("", "test.oc-adm-must-gather.")
		o.Expect(err).NotTo(o.HaveOccurred())
		defer os.RemoveAll(tempDir)
		o.Expect(oc.Run("adm", "must-gather").Args("--dest-dir", tempDir).Execute()).To(o.Succeed())

		pluginOutputDir := getPluginOutputDir(oc, tempDir)

		expectedDirectories := [][]string{
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io"},
			{pluginOutputDir, "cluster-scoped-resources", "operator.openshift.io"},
			{pluginOutputDir, "cluster-scoped-resources", "core"},
			{pluginOutputDir, "cluster-scoped-resources", "apiregistration.k8s.io"},
			{pluginOutputDir, "namespaces", "openshift"},
			{pluginOutputDir, "namespaces", "openshift-kube-apiserver-operator"},
		}

		expectedFiles := [][]string{
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "apiservers.yaml"},
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "authentications.yaml"},
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "builds.yaml"},
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "clusteroperators.yaml"},
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "clusterversions.yaml"},
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "consoles.yaml"},
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "dnses.yaml"},
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "featuregates.yaml"},
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "images.yaml"},
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "infrastructures.yaml"},
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "ingresses.yaml"},
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "networks.yaml"},
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "oauths.yaml"},
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "projects.yaml"},
			{pluginOutputDir, "cluster-scoped-resources", "config.openshift.io", "schedulers.yaml"},
			// TODO: This got broken and we need to fix this. Disabled temporarily.
			// {pluginOutputDir, "namespaces", "openshift-kube-apiserver", "core", "configmaps.yaml"},
			// {pluginOutputDir, "namespaces", "openshift-kube-apiserver", "core", "secrets.yaml"},
			{pluginOutputDir, "host_service_logs", "masters", "crio_service.log"},
			{pluginOutputDir, "host_service_logs", "masters", "kubelet_service.log"},
		}

		for _, expectedDirectory := range expectedDirectories {
			o.Expect(path.Join(expectedDirectory...)).To(o.BeADirectory())
		}

		emptyFiles := []string{}
		for _, expectedFile := range expectedFiles {
			expectedFilePath := path.Join(expectedFile...)
			o.Expect(expectedFilePath).To(o.BeAnExistingFile())
			stat, err := os.Stat(expectedFilePath)
			o.Expect(err).NotTo(o.HaveOccurred())
			if size := stat.Size(); size < 50 {
				emptyFiles = append(emptyFiles, expectedFilePath)
			}
		}
		if len(emptyFiles) > 0 {
			o.Expect(fmt.Errorf("expected files should not be empty: %s", strings.Join(emptyFiles, ","))).NotTo(o.HaveOccurred())
		}
	})

	g.It("runs successfully with options", func() {
		tempDir, err := ioutil.TempDir("", "test.oc-adm-must-gather.")
		o.Expect(err).NotTo(o.HaveOccurred())
		defer os.RemoveAll(tempDir)
		args := []string{
			"--dest-dir", tempDir,
			"--source-dir", "/artifacts",
			"--",
			"/bin/bash", "-c",
			"ls -l > /artifacts/ls.log",
		}
		o.Expect(oc.Run("adm", "must-gather").Args(args...).Execute()).To(o.Succeed())
		expectedFilePath := path.Join(getPluginOutputDir(oc, tempDir), "ls.log")
		o.Expect(expectedFilePath).To(o.BeAnExistingFile())
		stat, err := os.Stat(expectedFilePath)
		o.Expect(err).NotTo(o.HaveOccurred())
		o.Expect(stat.Size()).To(o.BeNumerically(">", 0))
	})

	g.It("runs successfully for audit logs", func() {
		// On IBM ROKS, events will not be part of the output, since audit logs do not include control plane logs.
		if e2e.TestContext.Provider == ibmcloud.ProviderName {
			g.Skip("ROKs doesn't have audit logs")
		}

		// makes some tokens that should not show in the audit logs
		const tokenName = "must-gather-audit-logs-token-plus-some-padding-here-to-make-the-limit"
		oauthClient := oauthv1client.NewForConfigOrDie(oc.AdminConfig())
		_, err1 := oauthClient.OAuthAccessTokens().Create(context.Background(), &oauthv1.OAuthAccessToken{
			ObjectMeta: metav1.ObjectMeta{
				Name: tokenName,
			},
			ClientName:  "openshift-challenging-client",
			ExpiresIn:   30,
			Scopes:      []string{"user:info"},
			RedirectURI: "https://127.0.0.1:12000/oauth/token/implicit",
			UserName:    "a",
			UserUID:     "1",
		}, metav1.CreateOptions{})
		o.Expect(err1).NotTo(o.HaveOccurred())
		_, err2 := oauthClient.OAuthAuthorizeTokens().Create(context.Background(), &oauthv1.OAuthAuthorizeToken{
			ObjectMeta: metav1.ObjectMeta{
				Name: tokenName,
			},
			ClientName:  "openshift-challenging-client",
			ExpiresIn:   30,
			Scopes:      []string{"user:info"},
			RedirectURI: "https://127.0.0.1:12000/oauth/token/implicit",
			UserName:    "a",
			UserUID:     "1",
		}, metav1.CreateOptions{})
		o.Expect(err2).NotTo(o.HaveOccurred())

		tempDir, err := ioutil.TempDir("", "test.oc-adm-must-gather.")
		o.Expect(err).NotTo(o.HaveOccurred())
		// defer os.RemoveAll(tempDir)

		args := []string{
			"--dest-dir", tempDir,
			"--",
			"/usr/bin/gather_audit_logs",
		}

		o.Expect(oc.Run("adm", "must-gather").Args(args...).Execute()).To(o.Succeed())
		// wait for the contents to show up in the plugin output directory, avoiding EOF errors
		time.Sleep(10 * time.Second)

		pluginOutputDir := getPluginOutputDir(oc, tempDir)

		expectedDirectoriesToExpectedCount := map[string]int{
			path.Join(pluginOutputDir, "audit_logs", "kube-apiserver"):      1000,
			path.Join(pluginOutputDir, "audit_logs", "openshift-apiserver"): 10, // openshift apiservers don't necessarily get much traffic.  Especially early in a run
		}

		expectedFiles := [][]string{
			{pluginOutputDir, "audit_logs", "kube-apiserver.audit_logs_listing"},
			{pluginOutputDir, "audit_logs", "openshift-apiserver.audit_logs_listing"},
		}

		// for some crazy reason, it seems that the files from must-gather take time to appear on disk for reading.  I don't understand why
		// but this was in a previous commit and I don't want to immediately flake: https://github.com/openshift/origin/commit/006745a535848e84dcbcdd1c83ae86deddd3a229#diff-ad1c47fa4213de16d8b3237df5d71724R168
		// so we're going to try to get a pass every 10 seconds for a minute.  If we pass, great.  If we don't, we report the
		// last error we had.
		var lastErr error
		err = wait.PollImmediate(10*time.Second, 1*time.Minute, func() (bool, error) {
			// make sure we do not log OAuth tokens
			for auditDirectory, expectedNumberOfAuditEntries := range expectedDirectoriesToExpectedCount {
				eventsChecked := 0
				err := filepath.Walk(auditDirectory, func(path string, info os.FileInfo, err error) error {
					g.By(path)
					o.Expect(err).NotTo(o.HaveOccurred())
					if info.IsDir() {
						return nil
					}

					if (strings.Contains(path, "-termination-") && strings.HasSuffix(path, ".log.gz")) || strings.HasSuffix(path, "termination.log.gz") {
						// these are expected, but have unstructured log format
						return nil
					}

					isAuditFile := (strings.Contains(path, "-audit-") && strings.HasSuffix(path, ".log.gz")) || strings.HasSuffix(path, "audit.log.gz")
					o.Expect(isAuditFile).To(o.BeTrue())

					// at this point, we expect only audit files with json events, one per line

					readFile := false

					file, err := os.Open(path)
					o.Expect(err).NotTo(o.HaveOccurred())
					defer file.Close()

					fi, err := file.Stat()
					o.Expect(err).NotTo(o.HaveOccurred())

					// it will happen that the audit files are sometimes empty, we can
					// safely ignore these files since they don't provide valuable information
					// TODO this doesn't seem right.  It should be really unlikely, but we'll deal with later
					if fi.Size() == 0 {
						return nil
					}

					gzipReader, err := gzip.NewReader(file)
					o.Expect(err).NotTo(o.HaveOccurred())

					scanner := bufio.NewScanner(gzipReader)
					for scanner.Scan() {
						text := scanner.Text()
						if !strings.HasSuffix(text, "}") {
							continue // ignore truncated data
						}
						o.Expect(text).To(o.HavePrefix(`{"kind":"Event",`))
						for _, token := range []string{"oauthaccesstokens", "oauthauthorizetokens", tokenName} {
							o.Expect(text).NotTo(o.ContainSubstring(token))
						}
						readFile = true
						eventsChecked++
					}
					// ignore this error as we usually fail to read the whole GZ file
					// o.Expect(scanner.Err()).NotTo(o.HaveOccurred())
					o.Expect(readFile).To(o.BeTrue())

					return nil
				})
				o.Expect(err).NotTo(o.HaveOccurred())

				if eventsChecked <= expectedNumberOfAuditEntries {
					lastErr = fmt.Errorf("expected %d audit events for %q, but only got %d", expectedNumberOfAuditEntries, auditDirectory, eventsChecked)
					return false, nil
				}

				// reset lastErr if we succeeded.
				lastErr = nil
			}

			// if we get here, it means both directories checked out ok
			return true, nil
		})
		o.Expect(lastErr).NotTo(o.HaveOccurred()) // print the last error first if we have one
		o.Expect(err).NotTo(o.HaveOccurred())     // otherwise be sure we fail on the timeout if it happened

		emptyFiles := []string{}
		for _, expectedFile := range expectedFiles {
			expectedFilePath := path.Join(expectedFile...)
			o.Expect(expectedFilePath).To(o.BeAnExistingFile())
			stat, err := os.Stat(expectedFilePath)
			o.Expect(err).NotTo(o.HaveOccurred())
			if size := stat.Size(); size < 50 {
				emptyFiles = append(emptyFiles, expectedFilePath)
			}
		}
		if len(emptyFiles) > 0 {
			o.Expect(fmt.Errorf("expected files should not be empty: %s", strings.Join(emptyFiles, ","))).NotTo(o.HaveOccurred())
		}
	})
})

func getPluginOutputDir(oc *exutil.CLI, tempDir string) string {
	imageClient := versioned.NewForConfigOrDie(oc.AdminConfig())
	stream, err := imageClient.ImageV1().ImageStreams("openshift").Get(context.Background(), "must-gather", metav1.GetOptions{})
	o.Expect(err).NotTo(o.HaveOccurred())
	imageId, ok := imageutil.ResolveLatestTaggedImage(stream, "latest")
	o.Expect(ok).To(o.BeTrue())
	pluginOutputDir := path.Join(tempDir, regexp.MustCompile("[^A-Za-z0-9]+").ReplaceAllString(imageId, "-"))
	fileInfo, err := os.Stat(pluginOutputDir)
	if err != nil || !fileInfo.IsDir() {
		pluginOutputDir = tempDir
	}
	return pluginOutputDir
}
<|endoftext|>


<commit_before>package irma

import (
	"encoding/xml"
	"fmt"

	"github.com/go-errors/errors"
	"github.com/privacybydesign/irmago/internal/fs"
)

// This file contains data types for scheme managers, issuers, credential types
// matching the XML files in irma_configuration.

// SchemeManager describes a scheme manager.
type SchemeManager struct {
	ID                string           `xml:"Id"`
	Name              TranslatedString `xml:"Name"`
	URL               string           `xml:"Url"`
	Contact           string           `xml:"contact"`
	Description       TranslatedString
	KeyshareServer    string
	KeyshareWebsite   string
	KeyshareAttribute string
	XMLVersion        int      `xml:"version,attr"`
	XMLName           xml.Name `xml:"SchemeManager"`

	Status SchemeManagerStatus `xml:"-"`
	Valid  bool                `xml:"-"` // true iff Status == SchemeManagerStatusValid

	Timestamp Timestamp

	index SchemeManagerIndex
}

// Issuer describes an issuer.
type Issuer struct {
	ID              string           `xml:"ID"`
	Name            TranslatedString `xml:"Name"`
	ShortName       TranslatedString `xml:"ShortName"`
	SchemeManagerID string           `xml:"SchemeManager"`
	ContactAddress  string
	ContactEMail    string
	URL             string `xml:"ContactAddress"`
	XMLVersion      int    `xml:"version,attr"`

	Valid bool `xml:"-"`
}

// CredentialType is a description of a credential type, specifying (a.o.) its name, issuer, and attributes.
type CredentialType struct {
	ID              string           `xml:"CredentialID"`
	Name            TranslatedString `xml:"Name"`
	ShortName       TranslatedString `xml:"ShortName"`
	IssuerID        string           `xml:"IssuerID"`
	SchemeManagerID string           `xml:"SchemeManager"`
	IsSingleton     bool             `xml:"ShouldBeSingleton"`
	Description     TranslatedString
	Attributes      []AttributeDescription `xml:"Attributes>Attribute"`
	XMLVersion      int                    `xml:"version,attr"`
	XMLName         xml.Name               `xml:"IssueSpecification"`
	IssueURL        TranslatedString       `xml:'IssueUrl'`

	Valid bool `xml:"-"`
}

// AttributeDescription is a description of an attribute within a credential type.
type AttributeDescription struct {
	ID           string `xml:"id,attr"`
	Optional     string `xml:"optional,attr"  json:",omitempty"`
	DisplayIndex *int   `xml:"displayIndex,attr" json:",omitempty"`
	Name         TranslatedString
	Description  TranslatedString
}

func (ad AttributeDescription) GetAttributeTypeIdentifier(cred CredentialTypeIdentifier) AttributeTypeIdentifier {
	return NewAttributeTypeIdentifier(cred.String() + "." + ad.ID)
}

func (ad AttributeDescription) IsOptional() bool {
	return ad.Optional == "true"
}

// ContainsAttribute tests whether the specified attribute is contained in this
// credentialtype.
func (ct *CredentialType) ContainsAttribute(ai AttributeTypeIdentifier) bool {
	if ai.CredentialTypeIdentifier().String() != ct.Identifier().String() {
		return false
	}
	for _, desc := range ct.Attributes {
		if desc.ID == ai.Name() {
			return true
		}
	}
	return false
}

// IndexOf returns the index of the specified attribute if present,
// or an error (and -1) if not present.
func (ct CredentialType) IndexOf(ai AttributeTypeIdentifier) (int, error) {
	if ai.CredentialTypeIdentifier() != ct.Identifier() {
		return -1, errors.New("Wrong credential type")
	}
	for i, description := range ct.Attributes {
		if description.ID == ai.Name() {
			return i, nil
		}
	}
	return -1, errors.New("Attribute identifier not found")
}

func (ct CredentialType) AttributeDescription(ai AttributeTypeIdentifier) *AttributeDescription {
	i, _ := ct.IndexOf(ai)
	if i == -1 {
		return nil
	}
	return &ct.Attributes[i]
}

// TranslatedString is a map of translated strings.
type TranslatedString map[string]string

type xmlTranslation struct {
	XMLName xml.Name
	Text    string `xml:",chardata"`
}

type xmlTranslatedString struct {
	Translations []xmlTranslation `xml:",any"`
}

// MarshalXML implements xml.Marshaler.
func (ts *TranslatedString) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
	temp := &xmlTranslatedString{}
	for lang, text := range *ts {
		temp.Translations = append(temp.Translations,
			xmlTranslation{XMLName: xml.Name{Local: lang}, Text: text},
		)
	}
	return e.EncodeElement(temp, start)
}

// UnmarshalXML unmarshals an XML tag containing a string translated to multiple languages,
// for example: <Foo><en>Hello world</en><nl>Hallo wereld</nl></Foo>
// into a TranslatedString: { "en": "Hello world" , "nl": "Hallo wereld" }
func (ts *TranslatedString) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
	if map[string]string(*ts) == nil {
		*ts = TranslatedString(make(map[string]string))
	}
	temp := &xmlTranslatedString{}
	if err := d.DecodeElement(temp, &start); err != nil {
		return err
	}
	for _, translation := range temp.Translations {
		(*ts)[translation.XMLName.Local] = translation.Text
	}
	return nil
}

// Identifier returns the identifier of the specified credential type.
func (ct *CredentialType) Identifier() CredentialTypeIdentifier {
	return NewCredentialTypeIdentifier(ct.SchemeManagerID + "." + ct.IssuerID + "." + ct.ID)
}

// IssuerIdentifier returns the issuer identifier of the specified credential type.
func (ct *CredentialType) IssuerIdentifier() IssuerIdentifier {
	return NewIssuerIdentifier(ct.SchemeManagerID + "." + ct.IssuerID)
}

func (ct *CredentialType) SchemeManagerIdentifier() SchemeManagerIdentifier {
	return NewSchemeManagerIdentifier(ct.SchemeManagerID)
}

func (ct *CredentialType) Logo(conf *Configuration) string {
	path := fmt.Sprintf("%s/%s/%s/Issues/%s/logo.png", conf.Path, ct.SchemeManagerID, ct.IssuerID, ct.ID)
	exists, err := fs.PathExists(path)
	if err != nil || !exists {
		return ""
	}
	return path
}

// Identifier returns the identifier of the specified issuer description.
func (id *Issuer) Identifier() IssuerIdentifier {
	return NewIssuerIdentifier(id.SchemeManagerID + "." + id.ID)
}

func (id *Issuer) SchemeManagerIdentifier() SchemeManagerIdentifier {
	return NewSchemeManagerIdentifier(id.SchemeManagerID)
}

func NewSchemeManager(name string) *SchemeManager {
	return &SchemeManager{ID: name, Status: SchemeManagerStatusUnprocessed, Valid: false}
}

// Identifier returns the identifier of the specified scheme manager.
func (sm *SchemeManager) Identifier() SchemeManagerIdentifier {
	return NewSchemeManagerIdentifier(sm.ID)
}

// Distributed indicates if this scheme manager uses a keyshare server.
func (sm *SchemeManager) Distributed() bool {
	return len(sm.KeyshareServer) > 0
}
<commit_msg>Remove superfluous tag from issuer description<commit_after>package irma

import (
	"encoding/xml"
	"fmt"

	"github.com/go-errors/errors"
	"github.com/privacybydesign/irmago/internal/fs"
)

// This file contains data types for scheme managers, issuers, credential types
// matching the XML files in irma_configuration.

// SchemeManager describes a scheme manager.
type SchemeManager struct {
	ID                string           `xml:"Id"`
	Name              TranslatedString `xml:"Name"`
	URL               string           `xml:"Url"`
	Contact           string           `xml:"contact"`
	Description       TranslatedString
	KeyshareServer    string
	KeyshareWebsite   string
	KeyshareAttribute string
	XMLVersion        int      `xml:"version,attr"`
	XMLName           xml.Name `xml:"SchemeManager"`

	Status SchemeManagerStatus `xml:"-"`
	Valid  bool                `xml:"-"` // true iff Status == SchemeManagerStatusValid

	Timestamp Timestamp

	index SchemeManagerIndex
}

// Issuer describes an issuer.
type Issuer struct {
	ID              string           `xml:"ID"`
	Name            TranslatedString `xml:"Name"`
	ShortName       TranslatedString `xml:"ShortName"`
	SchemeManagerID string           `xml:"SchemeManager"`
	ContactAddress  string
	ContactEMail    string
	XMLVersion      int `xml:"version,attr"`

	Valid bool `xml:"-"`
}

// CredentialType is a description of a credential type, specifying (a.o.) its name, issuer, and attributes.
type CredentialType struct {
	ID              string           `xml:"CredentialID"`
	Name            TranslatedString `xml:"Name"`
	ShortName       TranslatedString `xml:"ShortName"`
	IssuerID        string           `xml:"IssuerID"`
	SchemeManagerID string           `xml:"SchemeManager"`
	IsSingleton     bool             `xml:"ShouldBeSingleton"`
	Description     TranslatedString
	Attributes      []AttributeDescription `xml:"Attributes>Attribute"`
	XMLVersion      int                    `xml:"version,attr"`
	XMLName         xml.Name               `xml:"IssueSpecification"`
	IssueURL        TranslatedString       `xml:'IssueURL'`

	Valid bool `xml:"-"`
}

// AttributeDescription is a description of an attribute within a credential type.
type AttributeDescription struct {
	ID           string `xml:"id,attr"`
	Optional     string `xml:"optional,attr"  json:",omitempty"`
	DisplayIndex *int   `xml:"displayIndex,attr" json:",omitempty"`
	Name         TranslatedString
	Description  TranslatedString
}

func (ad AttributeDescription) GetAttributeTypeIdentifier(cred CredentialTypeIdentifier) AttributeTypeIdentifier {
	return NewAttributeTypeIdentifier(cred.String() + "." + ad.ID)
}

func (ad AttributeDescription) IsOptional() bool {
	return ad.Optional == "true"
}

// ContainsAttribute tests whether the specified attribute is contained in this
// credentialtype.
func (ct *CredentialType) ContainsAttribute(ai AttributeTypeIdentifier) bool {
	if ai.CredentialTypeIdentifier().String() != ct.Identifier().String() {
		return false
	}
	for _, desc := range ct.Attributes {
		if desc.ID == ai.Name() {
			return true
		}
	}
	return false
}

// IndexOf returns the index of the specified attribute if present,
// or an error (and -1) if not present.
func (ct CredentialType) IndexOf(ai AttributeTypeIdentifier) (int, error) {
	if ai.CredentialTypeIdentifier() != ct.Identifier() {
		return -1, errors.New("Wrong credential type")
	}
	for i, description := range ct.Attributes {
		if description.ID == ai.Name() {
			return i, nil
		}
	}
	return -1, errors.New("Attribute identifier not found")
}

func (ct CredentialType) AttributeDescription(ai AttributeTypeIdentifier) *AttributeDescription {
	i, _ := ct.IndexOf(ai)
	if i == -1 {
		return nil
	}
	return &ct.Attributes[i]
}

// TranslatedString is a map of translated strings.
type TranslatedString map[string]string

type xmlTranslation struct {
	XMLName xml.Name
	Text    string `xml:",chardata"`
}

type xmlTranslatedString struct {
	Translations []xmlTranslation `xml:",any"`
}

// MarshalXML implements xml.Marshaler.
func (ts *TranslatedString) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
	temp := &xmlTranslatedString{}
	for lang, text := range *ts {
		temp.Translations = append(temp.Translations,
			xmlTranslation{XMLName: xml.Name{Local: lang}, Text: text},
		)
	}
	return e.EncodeElement(temp, start)
}

// UnmarshalXML unmarshals an XML tag containing a string translated to multiple languages,
// for example: <Foo><en>Hello world</en><nl>Hallo wereld</nl></Foo>
// into a TranslatedString: { "en": "Hello world" , "nl": "Hallo wereld" }
func (ts *TranslatedString) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
	if map[string]string(*ts) == nil {
		*ts = TranslatedString(make(map[string]string))
	}
	temp := &xmlTranslatedString{}
	if err := d.DecodeElement(temp, &start); err != nil {
		return err
	}
	for _, translation := range temp.Translations {
		(*ts)[translation.XMLName.Local] = translation.Text
	}
	return nil
}

// Identifier returns the identifier of the specified credential type.
func (ct *CredentialType) Identifier() CredentialTypeIdentifier {
	return NewCredentialTypeIdentifier(ct.SchemeManagerID + "." + ct.IssuerID + "." + ct.ID)
}

// IssuerIdentifier returns the issuer identifier of the specified credential type.
func (ct *CredentialType) IssuerIdentifier() IssuerIdentifier {
	return NewIssuerIdentifier(ct.SchemeManagerID + "." + ct.IssuerID)
}

func (ct *CredentialType) SchemeManagerIdentifier() SchemeManagerIdentifier {
	return NewSchemeManagerIdentifier(ct.SchemeManagerID)
}

func (ct *CredentialType) Logo(conf *Configuration) string {
	path := fmt.Sprintf("%s/%s/%s/Issues/%s/logo.png", conf.Path, ct.SchemeManagerID, ct.IssuerID, ct.ID)
	exists, err := fs.PathExists(path)
	if err != nil || !exists {
		return ""
	}
	return path
}

// Identifier returns the identifier of the specified issuer description.
func (id *Issuer) Identifier() IssuerIdentifier {
	return NewIssuerIdentifier(id.SchemeManagerID + "." + id.ID)
}

func (id *Issuer) SchemeManagerIdentifier() SchemeManagerIdentifier {
	return NewSchemeManagerIdentifier(id.SchemeManagerID)
}

func NewSchemeManager(name string) *SchemeManager {
	return &SchemeManager{ID: name, Status: SchemeManagerStatusUnprocessed, Valid: false}
}

// Identifier returns the identifier of the specified scheme manager.
func (sm *SchemeManager) Identifier() SchemeManagerIdentifier {
	return NewSchemeManagerIdentifier(sm.ID)
}

// Distributed indicates if this scheme manager uses a keyshare server.
func (sm *SchemeManager) Distributed() bool {
	return len(sm.KeyshareServer) > 0
}
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package app

import (
	"context"
	"fmt"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/globalsign/mgo"
	"github.com/globalsign/mgo/bson"
	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/tsuru/config"
	"github.com/tsuru/tsuru/api/shutdown"
	"github.com/tsuru/tsuru/db"
	"github.com/tsuru/tsuru/log"
	"golang.org/x/time/rate"
)

var (
	bulkMaxWaitMongoTime = 1 * time.Second
	bulkMaxNumberMsgs    = 1000
	bulkQueueMaxSize     = 10000

	rateLimitWarningInterval = 5 * time.Second

	buckets = append([]float64{0.1, 0.5}, prometheus.ExponentialBuckets(1, 1.6, 15)...)

	logsInQueue = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "tsuru_logs_queue_current",
		Help: "The current number of log entries in dispatcher queue.",
	})

	logsInAppQueues = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "tsuru_logs_app_queues_current",
		Help: "The current number of log entries in app queues.",
	}, []string{"app"})

	logsQueueBlockedTotal = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsuru_logs_queue_blocked_seconds_total",
		Help: "The total time spent blocked trying to add log to queue.",
	})

	logsQueueSize = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "tsuru_logs_dispatcher_queue_size",
		Help: "The max number of log entries in a dispatcher queue.",
	})

	logsEnqueued = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "tsuru_logs_enqueued_total",
		Help: "The number of log entries enqueued for processing.",
	}, []string{"app"})

	logsWritten = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "tsuru_logs_write_total",
		Help: "The number of log entries written to mongo.",
	}, []string{"app"})

	logsDropped = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "tsuru_logs_dropped_total",
		Help: "The number of log entries dropped due to full buffers.",
	}, []string{"app"})

	logsDroppedRateLimit = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "tsuru_logs_dropped_rate_limit_total",
		Help: "The number of log entries dropped due to rate limit exceeded.",
	}, []string{"app"})

	logsMongoFullLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "tsuru_logs_mongo_full_duration_seconds",
		Help:    "The latency distributions for log messages to be stored in database.",
		Buckets: buckets,
	})

	logsMongoLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "tsuru_logs_mongo_duration_seconds",
		Help:    "The latency distributions for log messages to be stored in database.",
		Buckets: buckets,
	})
)

func init() {
	prometheus.MustRegister(logsInQueue)
	prometheus.MustRegister(logsInAppQueues)
	prometheus.MustRegister(logsQueueSize)
	prometheus.MustRegister(logsEnqueued)
	prometheus.MustRegister(logsWritten)
	prometheus.MustRegister(logsDropped)
	prometheus.MustRegister(logsDroppedRateLimit)
	prometheus.MustRegister(logsQueueBlockedTotal)
	prometheus.MustRegister(logsMongoFullLatency)
	prometheus.MustRegister(logsMongoLatency)
}

type LogListener struct {
	c       <-chan Applog
	logConn *db.LogStorage
	quit    chan struct{}
}

func isCappedPositionLost(err error) bool {
	if err == nil {
		return false
	}
	return strings.Contains(err.Error(), "CappedPositionLost")
}

func isSessionClosed(r interface{}) bool {
	return fmt.Sprintf("%v", r) == "Session already closed"
}

func NewLogListener(a *App, filterLog Applog) (*LogListener, error) {
	conn, err := db.LogConn()
	if err != nil {
		return nil, err
	}
	c := make(chan Applog, 10)
	quit := make(chan struct{})
	coll := conn.AppLogCollection(a.Name)
	var lastLog Applog
	err = coll.Find(nil).Sort("-_id").Limit(1).One(&lastLog)
	if err == mgo.ErrNotFound {
		// Tail cursors do not work correctly if the collection is empty (the
		// Next() call wouldn't block). So if the collection is empty we insert
		// the very first log line in it. This is quite rare in the real world
		// though so the impact of this extra log message is really small.
		err = a.Log("Logs initialization", "tsuru", "")
		if err != nil {
			return nil, err
		}
		err = coll.Find(nil).Sort("-_id").Limit(1).One(&lastLog)
	}
	if err != nil {
		return nil, err
	}
	lastId := lastLog.MongoID
	mkQuery := func() bson.M {
		m := bson.M{
			"_id": bson.M{"$gt": lastId},
		}
		if filterLog.Source != "" {
			m["source"] = filterLog.Source
		}
		if filterLog.Unit != "" {
			m["unit"] = filterLog.Unit
		}
		return m
	}
	query := coll.Find(mkQuery())
	tailTimeout := 10 * time.Second
	iter := query.Sort("$natural").Tail(tailTimeout)
	go func() {
		defer close(c)
		defer func() {
			if r := recover(); r != nil {
				if isSessionClosed(r) {
					return
				}
				panic(err)
			}
		}()
		for {
			var applog Applog
			for iter.Next(&applog) {
				lastId = applog.MongoID
				select {
				case c <- applog:
				case <-quit:
					iter.Close()
					return
				}
			}
			if iter.Timeout() {
				continue
			}
			if err := iter.Err(); err != nil {
				if !isCappedPositionLost(err) {
					log.Errorf("error tailing logs: %v", err)
					iter.Close()
					return
				}
			}
			iter.Close()
			query = coll.Find(mkQuery())
			iter = query.Sort("$natural").Tail(tailTimeout)
		}
	}()
	l := LogListener{c: c, logConn: conn, quit: quit}
	return &l, nil
}

func (l *LogListener) ListenChan() <-chan Applog {
	return l.c
}

func (l *LogListener) Close() {
	l.logConn.Close()
	if l.quit != nil {
		close(l.quit)
		l.quit = nil
	}
}

type LogDispatcher struct {
	mu             sync.RWMutex
	dispatchers    map[string]*appLogDispatcher
	msgCh          chan *msgWithTS
	shuttingDown   int32
	doneProcessing chan struct{}
}

type msgWithTS struct {
	msg        *Applog
	arriveTime time.Time
}

func NewlogDispatcher(chanSize int) *LogDispatcher {
	d := &LogDispatcher{
		dispatchers:    make(map[string]*appLogDispatcher),
		msgCh:          make(chan *msgWithTS, chanSize),
		doneProcessing: make(chan struct{}),
	}
	go d.runWriter()
	shutdown.Register(d)
	logsQueueSize.Set(float64(chanSize))
	return d
}

func (d *LogDispatcher) getMessageDispatcher(msg *Applog) *appLogDispatcher {
	appName := msg.AppName
	d.mu.RLock()
	appD, ok := d.dispatchers[appName]
	if !ok {
		d.mu.RUnlock()
		d.mu.Lock()
		appD, ok = d.dispatchers[appName]
		if !ok {
			appD = newAppLogDispatcher(appName)
			d.dispatchers[appName] = appD
		}
		d.mu.Unlock()
	} else {
		d.mu.RUnlock()
	}
	return appD
}

func (d *LogDispatcher) runWriter() {
	defer close(d.doneProcessing)
	for msgExtra := range d.msgCh {
		if msgExtra == nil {
			break
		}
		logsInQueue.Dec()
		appD := d.getMessageDispatcher(msgExtra.msg)
		appD.send(msgExtra)
	}
}

func (d *LogDispatcher) Send(msg *Applog) error {
	if atomic.LoadInt32(&d.shuttingDown) == 1 {
		return errors.New("log dispatcher is shutting down")
	}
	logsInQueue.Inc()
	logsEnqueued.WithLabelValues(msg.AppName).Inc()
	msgExtra := &msgWithTS{msg: msg, arriveTime: time.Now()}
	select {
	case d.msgCh <- msgExtra:
	default:
		t0 := time.Now()
		d.msgCh <- msgExtra
		logsQueueBlockedTotal.Add(time.Since(t0).Seconds())
	}
	return nil
}

func (a *LogDispatcher) String() string {
	return "log dispatcher"
}

func (d *LogDispatcher) Shutdown(ctx context.Context) error {
	atomic.StoreInt32(&d.shuttingDown, 1)
	d.msgCh <- nil
	<-d.doneProcessing
	logsInQueue.Set(0)
	for _, appD := range d.dispatchers {
		appD.stopWait()
	}
	return nil
}

type appLogDispatcher struct {
	appName string
	*bulkProcessor
}

func newAppLogDispatcher(appName string) *appLogDispatcher {
	d := &appLogDispatcher{
		bulkProcessor: initBulkProcessor(bulkMaxWaitMongoTime, bulkMaxNumberMsgs, appName),
		appName:       appName,
	}
	d.flushable = d
	go d.run()
	return d
}

func (d *appLogDispatcher) flush(msgs []interface{}, lastMessage *msgWithTS) bool {
	conn, err := db.LogConn()
	if err != nil {
		log.Errorf("[log flusher] unable to connect to mongodb: %s", err)
		return false
	}
	coll := conn.AppLogCollection(d.appName)
	err = coll.Insert(msgs...)
	coll.Close()
	if err != nil {
		log.Errorf("[log flusher] unable to insert logs: %s", err)
		return false
	}
	if lastMessage != nil {
		logsMongoLatency.Observe(time.Since(lastMessage.arriveTime).Seconds())
		logsMongoFullLatency.Observe(time.Since(lastMessage.msg.Date).Seconds())
	}
	logsWritten.WithLabelValues(d.appName).Add(float64(len(msgs)))
	return true
}

type bulkProcessor struct {
	appName     string
	maxWaitTime time.Duration
	bulkSize    int
	finished    chan struct{}
	ch          chan *msgWithTS
	nextNotify  *time.Timer
	flushable   interface {
		flush([]interface{}, *msgWithTS) bool
	}
}

func initBulkProcessor(maxWait time.Duration, bulkSize int, appName string) *bulkProcessor {
	queueSize, err := config.GetInt("log:queue-size")
	if err != nil || queueSize == 0 {
		queueSize = bulkQueueMaxSize
	}
	return &bulkProcessor{
		appName:     appName,
		maxWaitTime: maxWait,
		bulkSize:    bulkSize,
		finished:    make(chan struct{}),
		ch:          make(chan *msgWithTS, queueSize),
		nextNotify:  time.NewTimer(0),
	}
}

func (p *bulkProcessor) send(msg *msgWithTS) {
	select {
	case p.ch <- msg:
		logsInAppQueues.WithLabelValues(p.appName).Set(float64(len(p.ch)))
	default:
		logsDropped.WithLabelValues(p.appName).Inc()
		select {
		case <-p.nextNotify.C:
			log.Errorf("dropping log messages to mongodb due to full channel buffer. app: %q, len: %d", msg.msg.AppName, len(p.ch))
			p.nextNotify.Reset(time.Minute)
		default:
		}
	}
}

func (p *bulkProcessor) stopWait() {
	p.ch <- nil
	<-p.finished
}

func (p *bulkProcessor) rateLimitWarning(rateLimit int) *Applog {
	return &Applog{
		AppName: p.appName,
		Date:    time.Now(),
		Message: fmt.Sprintf("Log messages dropped due to exceeded rate limit. Limit: %v logs/s.", rateLimit),
		Source:  "tsuru",
		Unit:    "api",
	}
}

func updateLogRateLimiter(rateLimiter *rate.Limiter) *rate.Limiter {
	rateLimit, _ := config.GetInt("log:app-log-rate-limit")
	if rateLimit <= 0 {
		return nil
	}
	if rateLimiter == nil || rateLimiter.Burst() != rateLimit {
		return rate.NewLimiter(rate.Limit(rateLimit), rateLimit)
	}
	return rateLimiter
}

func (p *bulkProcessor) run() {
	defer close(p.finished)
	t := time.NewTimer(p.maxWaitTime)
	pos := 0
	bulkBuffer := make([]interface{}, p.bulkSize)
	shouldReturn := false
	var lastMessage *msgWithTS
	var lastRateNotice time.Time
	logsInAppQueue := logsInAppQueues.WithLabelValues(p.appName)
	logsDropped := logsDroppedRateLimit.WithLabelValues(p.appName)
	rateLimiter := updateLogRateLimiter(nil)
	for {
		var flush bool
		select {
		case msgExtra := <-p.ch:
			logsInAppQueue.Set(float64(len(p.ch)))
			if msgExtra == nil {
				flush = true
				shouldReturn = true
				break
			}
			if pos == p.bulkSize {
				flush = true
				break
			}

			if rateLimiter != nil && !rateLimiter.Allow() {
				logsDropped.Inc()
				if time.Since(lastRateNotice) > rateLimitWarningInterval {
					lastRateNotice = time.Now()
					bulkBuffer[pos] = p.rateLimitWarning(rateLimiter.Burst())
					pos++
				}
				flush = p.bulkSize == pos
				break
			}

			lastMessage = msgExtra
			bulkBuffer[pos] = msgExtra.msg
			pos++
			flush = p.bulkSize == pos
		case <-t.C:
			flush = true
			t.Reset(p.maxWaitTime)
		}
		if flush && pos > 0 {
			if p.flushable.flush(bulkBuffer[:pos], lastMessage) {
				lastMessage = nil
				pos = 0
			}
			rateLimiter = updateLogRateLimiter(rateLimiter)
		}
		if shouldReturn {
			return
		}
	}
}
<commit_msg>app: fix leaked log db connections on log listener errors<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package app

import (
	"context"
	"fmt"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/globalsign/mgo"
	"github.com/globalsign/mgo/bson"
	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/tsuru/config"
	"github.com/tsuru/tsuru/api/shutdown"
	"github.com/tsuru/tsuru/db"
	"github.com/tsuru/tsuru/log"
	"golang.org/x/time/rate"
)

var (
	bulkMaxWaitMongoTime = 1 * time.Second
	bulkMaxNumberMsgs    = 1000
	bulkQueueMaxSize     = 10000

	rateLimitWarningInterval = 5 * time.Second

	buckets = append([]float64{0.1, 0.5}, prometheus.ExponentialBuckets(1, 1.6, 15)...)

	logsInQueue = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "tsuru_logs_queue_current",
		Help: "The current number of log entries in dispatcher queue.",
	})

	logsInAppQueues = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "tsuru_logs_app_queues_current",
		Help: "The current number of log entries in app queues.",
	}, []string{"app"})

	logsQueueBlockedTotal = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsuru_logs_queue_blocked_seconds_total",
		Help: "The total time spent blocked trying to add log to queue.",
	})

	logsQueueSize = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "tsuru_logs_dispatcher_queue_size",
		Help: "The max number of log entries in a dispatcher queue.",
	})

	logsEnqueued = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "tsuru_logs_enqueued_total",
		Help: "The number of log entries enqueued for processing.",
	}, []string{"app"})

	logsWritten = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "tsuru_logs_write_total",
		Help: "The number of log entries written to mongo.",
	}, []string{"app"})

	logsDropped = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "tsuru_logs_dropped_total",
		Help: "The number of log entries dropped due to full buffers.",
	}, []string{"app"})

	logsDroppedRateLimit = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "tsuru_logs_dropped_rate_limit_total",
		Help: "The number of log entries dropped due to rate limit exceeded.",
	}, []string{"app"})

	logsMongoFullLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "tsuru_logs_mongo_full_duration_seconds",
		Help:    "The latency distributions for log messages to be stored in database.",
		Buckets: buckets,
	})

	logsMongoLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "tsuru_logs_mongo_duration_seconds",
		Help:    "The latency distributions for log messages to be stored in database.",
		Buckets: buckets,
	})
)

func init() {
	prometheus.MustRegister(logsInQueue)
	prometheus.MustRegister(logsInAppQueues)
	prometheus.MustRegister(logsQueueSize)
	prometheus.MustRegister(logsEnqueued)
	prometheus.MustRegister(logsWritten)
	prometheus.MustRegister(logsDropped)
	prometheus.MustRegister(logsDroppedRateLimit)
	prometheus.MustRegister(logsQueueBlockedTotal)
	prometheus.MustRegister(logsMongoFullLatency)
	prometheus.MustRegister(logsMongoLatency)
}

type LogListener struct {
	c       <-chan Applog
	logConn *db.LogStorage
	quit    chan struct{}
}

func isCappedPositionLost(err error) bool {
	if err == nil {
		return false
	}
	return strings.Contains(err.Error(), "CappedPositionLost")
}

func isSessionClosed(r interface{}) bool {
	return fmt.Sprintf("%v", r) == "Session already closed"
}

func NewLogListener(a *App, filterLog Applog) (*LogListener, error) {
	conn, err := db.LogConn()
	if err != nil {
		return nil, err
	}
	c := make(chan Applog, 10)
	quit := make(chan struct{})
	coll := conn.AppLogCollection(a.Name)
	var lastLog Applog
	err = coll.Find(nil).Sort("-_id").Limit(1).One(&lastLog)
	if err == mgo.ErrNotFound {
		// Tail cursors do not work correctly if the collection is empty (the
		// Next() call wouldn't block). So if the collection is empty we insert
		// the very first log line in it. This is quite rare in the real world
		// though so the impact of this extra log message is really small.
		err = a.Log("Logs initialization", "tsuru", "")
		if err != nil {
			conn.Close()
			return nil, err
		}
		err = coll.Find(nil).Sort("-_id").Limit(1).One(&lastLog)
	}
	if err != nil {
		conn.Close()
		return nil, err
	}
	lastId := lastLog.MongoID
	mkQuery := func() bson.M {
		m := bson.M{
			"_id": bson.M{"$gt": lastId},
		}
		if filterLog.Source != "" {
			m["source"] = filterLog.Source
		}
		if filterLog.Unit != "" {
			m["unit"] = filterLog.Unit
		}
		return m
	}
	query := coll.Find(mkQuery())
	tailTimeout := 10 * time.Second
	iter := query.Sort("$natural").Tail(tailTimeout)
	go func() {
		defer close(c)
		defer func() {
			if r := recover(); r != nil {
				if isSessionClosed(r) {
					return
				}
				panic(err)
			}
		}()
		for {
			var applog Applog
			for iter.Next(&applog) {
				lastId = applog.MongoID
				select {
				case c <- applog:
				case <-quit:
					iter.Close()
					return
				}
			}
			if iter.Timeout() {
				continue
			}
			if err := iter.Err(); err != nil {
				if !isCappedPositionLost(err) {
					log.Errorf("error tailing logs: %v", err)
					iter.Close()
					return
				}
			}
			iter.Close()
			query = coll.Find(mkQuery())
			iter = query.Sort("$natural").Tail(tailTimeout)
		}
	}()
	l := LogListener{c: c, logConn: conn, quit: quit}
	return &l, nil
}

func (l *LogListener) ListenChan() <-chan Applog {
	return l.c
}

func (l *LogListener) Close() {
	l.logConn.Close()
	if l.quit != nil {
		close(l.quit)
		l.quit = nil
	}
}

type LogDispatcher struct {
	mu             sync.RWMutex
	dispatchers    map[string]*appLogDispatcher
	msgCh          chan *msgWithTS
	shuttingDown   int32
	doneProcessing chan struct{}
}

type msgWithTS struct {
	msg        *Applog
	arriveTime time.Time
}

func NewlogDispatcher(chanSize int) *LogDispatcher {
	d := &LogDispatcher{
		dispatchers:    make(map[string]*appLogDispatcher),
		msgCh:          make(chan *msgWithTS, chanSize),
		doneProcessing: make(chan struct{}),
	}
	go d.runWriter()
	shutdown.Register(d)
	logsQueueSize.Set(float64(chanSize))
	return d
}

func (d *LogDispatcher) getMessageDispatcher(msg *Applog) *appLogDispatcher {
	appName := msg.AppName
	d.mu.RLock()
	appD, ok := d.dispatchers[appName]
	if !ok {
		d.mu.RUnlock()
		d.mu.Lock()
		appD, ok = d.dispatchers[appName]
		if !ok {
			appD = newAppLogDispatcher(appName)
			d.dispatchers[appName] = appD
		}
		d.mu.Unlock()
	} else {
		d.mu.RUnlock()
	}
	return appD
}

func (d *LogDispatcher) runWriter() {
	defer close(d.doneProcessing)
	for msgExtra := range d.msgCh {
		if msgExtra == nil {
			break
		}
		logsInQueue.Dec()
		appD := d.getMessageDispatcher(msgExtra.msg)
		appD.send(msgExtra)
	}
}

func (d *LogDispatcher) Send(msg *Applog) error {
	if atomic.LoadInt32(&d.shuttingDown) == 1 {
		return errors.New("log dispatcher is shutting down")
	}
	logsInQueue.Inc()
	logsEnqueued.WithLabelValues(msg.AppName).Inc()
	msgExtra := &msgWithTS{msg: msg, arriveTime: time.Now()}
	select {
	case d.msgCh <- msgExtra:
	default:
		t0 := time.Now()
		d.msgCh <- msgExtra
		logsQueueBlockedTotal.Add(time.Since(t0).Seconds())
	}
	return nil
}

func (a *LogDispatcher) String() string {
	return "log dispatcher"
}

func (d *LogDispatcher) Shutdown(ctx context.Context) error {
	atomic.StoreInt32(&d.shuttingDown, 1)
	d.msgCh <- nil
	<-d.doneProcessing
	logsInQueue.Set(0)
	for _, appD := range d.dispatchers {
		appD.stopWait()
	}
	return nil
}

type appLogDispatcher struct {
	appName string
	*bulkProcessor
}

func newAppLogDispatcher(appName string) *appLogDispatcher {
	d := &appLogDispatcher{
		bulkProcessor: initBulkProcessor(bulkMaxWaitMongoTime, bulkMaxNumberMsgs, appName),
		appName:       appName,
	}
	d.flushable = d
	go d.run()
	return d
}

func (d *appLogDispatcher) flush(msgs []interface{}, lastMessage *msgWithTS) bool {
	conn, err := db.LogConn()
	if err != nil {
		log.Errorf("[log flusher] unable to connect to mongodb: %s", err)
		return false
	}
	coll := conn.AppLogCollection(d.appName)
	err = coll.Insert(msgs...)
	coll.Close()
	if err != nil {
		log.Errorf("[log flusher] unable to insert logs: %s", err)
		return false
	}
	if lastMessage != nil {
		logsMongoLatency.Observe(time.Since(lastMessage.arriveTime).Seconds())
		logsMongoFullLatency.Observe(time.Since(lastMessage.msg.Date).Seconds())
	}
	logsWritten.WithLabelValues(d.appName).Add(float64(len(msgs)))
	return true
}

type bulkProcessor struct {
	appName     string
	maxWaitTime time.Duration
	bulkSize    int
	finished    chan struct{}
	ch          chan *msgWithTS
	nextNotify  *time.Timer
	flushable   interface {
		flush([]interface{}, *msgWithTS) bool
	}
}

func initBulkProcessor(maxWait time.Duration, bulkSize int, appName string) *bulkProcessor {
	queueSize, err := config.GetInt("log:queue-size")
	if err != nil || queueSize == 0 {
		queueSize = bulkQueueMaxSize
	}
	return &bulkProcessor{
		appName:     appName,
		maxWaitTime: maxWait,
		bulkSize:    bulkSize,
		finished:    make(chan struct{}),
		ch:          make(chan *msgWithTS, queueSize),
		nextNotify:  time.NewTimer(0),
	}
}

func (p *bulkProcessor) send(msg *msgWithTS) {
	select {
	case p.ch <- msg:
		logsInAppQueues.WithLabelValues(p.appName).Set(float64(len(p.ch)))
	default:
		logsDropped.WithLabelValues(p.appName).Inc()
		select {
		case <-p.nextNotify.C:
			log.Errorf("dropping log messages to mongodb due to full channel buffer. app: %q, len: %d", msg.msg.AppName, len(p.ch))
			p.nextNotify.Reset(time.Minute)
		default:
		}
	}
}

func (p *bulkProcessor) stopWait() {
	p.ch <- nil
	<-p.finished
}

func (p *bulkProcessor) rateLimitWarning(rateLimit int) *Applog {
	return &Applog{
		AppName: p.appName,
		Date:    time.Now(),
		Message: fmt.Sprintf("Log messages dropped due to exceeded rate limit. Limit: %v logs/s.", rateLimit),
		Source:  "tsuru",
		Unit:    "api",
	}
}

func updateLogRateLimiter(rateLimiter *rate.Limiter) *rate.Limiter {
	rateLimit, _ := config.GetInt("log:app-log-rate-limit")
	if rateLimit <= 0 {
		return nil
	}
	if rateLimiter == nil || rateLimiter.Burst() != rateLimit {
		return rate.NewLimiter(rate.Limit(rateLimit), rateLimit)
	}
	return rateLimiter
}

func (p *bulkProcessor) run() {
	defer close(p.finished)
	t := time.NewTimer(p.maxWaitTime)
	pos := 0
	bulkBuffer := make([]interface{}, p.bulkSize)
	shouldReturn := false
	var lastMessage *msgWithTS
	var lastRateNotice time.Time
	logsInAppQueue := logsInAppQueues.WithLabelValues(p.appName)
	logsDropped := logsDroppedRateLimit.WithLabelValues(p.appName)
	rateLimiter := updateLogRateLimiter(nil)
	for {
		var flush bool
		select {
		case msgExtra := <-p.ch:
			logsInAppQueue.Set(float64(len(p.ch)))
			if msgExtra == nil {
				flush = true
				shouldReturn = true
				break
			}
			if pos == p.bulkSize {
				flush = true
				break
			}

			if rateLimiter != nil && !rateLimiter.Allow() {
				logsDropped.Inc()
				if time.Since(lastRateNotice) > rateLimitWarningInterval {
					lastRateNotice = time.Now()
					bulkBuffer[pos] = p.rateLimitWarning(rateLimiter.Burst())
					pos++
				}
				flush = p.bulkSize == pos
				break
			}

			lastMessage = msgExtra
			bulkBuffer[pos] = msgExtra.msg
			pos++
			flush = p.bulkSize == pos
		case <-t.C:
			flush = true
			t.Reset(p.maxWaitTime)
		}
		if flush && pos > 0 {
			if p.flushable.flush(bulkBuffer[:pos], lastMessage) {
				lastMessage = nil
				pos = 0
			}
			rateLimiter = updateLogRateLimiter(rateLimiter)
		}
		if shouldReturn {
			return
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"html/template"
	"log"
	"net/http"
	"strconv"

	"github.com/google/uuid"
)

var templFuncs = template.FuncMap{
	"prettyDisplay": prettyDisplay,
	"delta":         computeDelta,
}
var templ = template.Must(template.New("").Funcs(templFuncs).ParseGlob("templates/*.html"))

func prettyDisplay(cents int) string {
	return fmt.Sprintf("%.02f", float64(cents)/100)
}

func computeDelta(balance, target int) []string {
	delta := balance - target
	cls := "delta-ok"
	if delta < 0 {
		cls = "delta-warn"
	}
	return []string{cls, fmt.Sprintf(`%.02f`, float64(delta)/100)}
}

func handleDeleteRequest(db *DB, w http.ResponseWriter, r *http.Request) {
	log.Printf(`delete: %v`, r.URL)
	log.Printf(`id: %s`, r.FormValue("id"))

	id, err := uuid.Parse(r.FormValue("id"))
	if err != nil {
		log.Printf(`update: can't parse ID: %s`, err)
		http.Redirect(w, r, "/", http.StatusSeeOther)
		return
	}

	db.DeleteEnvelope(id)

	http.Redirect(w, r, "/", http.StatusSeeOther)
}

func handleUpdateRequest(db *DB, w http.ResponseWriter, r *http.Request) {
	log.Printf(`update: %v`, r.URL)

	log.Printf(`name: %s`, r.FormValue("env-name"))
	log.Printf(`target: %s`, r.FormValue("env-target"))
	log.Printf(`monthtarget: %s`, r.FormValue("env-monthtarget"))
	log.Printf(`return: %s`, r.FormValue("env-return"))

	returnTo := "/"
	if r.FormValue("env-return") != "" {
		returnTo = "/details?id=" + r.FormValue("env-return")
	}

	id, err := uuid.Parse(r.FormValue("env-id"))
	if err != nil {
		log.Printf(`update: can't parse ID: %s`, err)
		id = uuid.New()
	}

	name := r.FormValue("env-name")

	newTarget := 0
	if tgt, err := strconv.ParseFloat(r.FormValue("env-target"), 64); err == nil {
		newTarget = int(tgt * 100)
	}

	newMonthTarget := 0
	if monthtgt, err := strconv.ParseFloat(r.FormValue("env-monthtarget"), 64); err == nil {
		newMonthTarget = int(monthtgt * 100)
	}

	if err = db.UpdateEnvelopeMeta(id, name, newTarget, newMonthTarget); err != nil {
		log.Printf(`can't update envelope %s: %s`, id, err)
		http.Redirect(w, r, returnTo, http.StatusSeeOther)
		return
	}

	http.Redirect(w, r, returnTo, http.StatusSeeOther)
}

func handleDebug(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Content-Type", "text/plain")
	w.Write([]byte("Hic sunt dracones\r\n\r\n"))
}

func handleDetail(db *DB, w http.ResponseWriter, r *http.Request) {
	log.Printf(`handling detail for id %s`, r.FormValue("id"))
	id, err := uuid.Parse(r.FormValue("id"))
	if err != nil {
		log.Printf(`detail: can't parse ID: %s`, err)
		http.Redirect(w, r, "/", http.StatusSeeOther)
		return
	}

	e, events, err := db.EnvelopeWithHistory(id)
	if err != nil {
		log.Printf(`detail: can't get envelope and history from DB: %s`, err)
		http.Redirect(w, r, "/", http.StatusSeeOther)
		return
	}

	events_rev := []Event{}
	for idx := len(events) - 1; idx >= 0; idx-- {
		events_rev = append(events_rev, events[idx])
	}

	param := struct {
		Envelope *Envelope
		Events   []Event
	}{e, events_rev}

	if err := templ.ExecuteTemplate(w, "details.html", param); err != nil {
		log.Printf(`error rendering details template: %s`, err)
	}
}

func handleTx(db *DB, w http.ResponseWriter, r *http.Request) {
	log.Printf(`tx for id %s`, r.FormValue(`id`))

	id, err := uuid.Parse(r.FormValue("id"))
	if err != nil {
		log.Printf(`tx: can't parse ID: %s`, err)
		http.Redirect(w, r, "/", http.StatusSeeOther)
		return
	}

	dir := r.FormValue(`dir`)
	switch dir {
	case `in`:
		/* nothing */
	case `out`:
		/* nothing */
	case `inout`:
		/* nothing */
	default:
		dir = `inout`
	}

	env, err := db.Envelope(id)
	if err != nil {
		log.Printf(`tx: can't get envelope %s: %s`, r.FormValue(`id`), err)
		http.Redirect(w, r, "/", http.StatusSeeOther)
		return
	}

	if r.Method != "POST" {
		switch dir {
		case `in`:
			fallthrough
		case `out`:
			log.Printf(`tx %s`, dir)
			params := struct{
				Envelope *Envelope
				Direction string
			}{
				Envelope: env,
				Direction: dir,
			}
			if err := templ.ExecuteTemplate(w, "transfer_in.html", params); err != nil {
				log.Printf(`error rendering details template: %s`, err)
			}
		default:
			params := struct{
				AllEnvelopes []*Envelope
				This *Envelope
			}{
				AllEnvelopes: []*Envelope{},
				This: env,
			}
			for _, e := range db.AllEnvelopes() {
				if e.Id != env.Id {
					params.AllEnvelopes = append(params.AllEnvelopes, e)
				}
			}
			if err := templ.ExecuteTemplate(w, "transfer.html", params); err != nil {
				log.Printf(`error rendering details template: %s`, err)
			}
			log.Printf(`tx inout`)
		}
	} else {
		log.Printf(`updating env %s`, r.FormValue(`id`))
		log.Printf(`  amount: %s`, r.FormValue(`amount`))
		amount, err := strconv.ParseFloat(r.FormValue(`amount`), 64)
		if err != nil {
			log.Printf(`can't parse %s: %s`, r.FormValue(`amount`), err)
			http.Redirect(w, r, "/", http.StatusSeeOther)
			return
		}
		switch dir {
		case `in`:
			if err = db.UpdateEnvelopeBalance(id, int(amount * 100)); err != nil {
				log.Printf(`can't update balance: %s`, err)
			}
		case `out`:
			if err = db.UpdateEnvelopeBalance(id, -int(amount * 100)); err != nil {
				log.Printf(`can't update balance: %s`, err)
			}
		default:
			destId, err := uuid.Parse(r.FormValue("destination"))
			if err != nil {
				log.Printf(`tx: can't parse ID: %s`, err)
				http.Redirect(w, r, "/", http.StatusSeeOther)
				return
			}
			if err = db.UpdateEnvelopeBalance(destId, int(amount * 100)); err != nil {
				log.Printf(`can't update balance: %s`, err)
			} else if err = db.UpdateEnvelopeBalance(id, -int(amount * 100)); err != nil {
				log.Printf(`can't update balance: %s`, err)
			}
			http.Redirect(w, r, fmt.Sprintf("/details?id=%s", destId), http.StatusSeeOther)
			return
		}
		http.Redirect(w, r, fmt.Sprintf("/details?id=%s", r.FormValue(`id`)), http.StatusSeeOther)
		return
	}
}

func handleSpread(db *DB, w http.ResponseWriter, r *http.Request) {
	log.Printf(`handling spread for id %s`, r.FormValue("id"))

	id, err := uuid.Parse(r.FormValue("id"))
	if err != nil {
		log.Printf(`spread: can't parse ID: %s`, err)
		http.Redirect(w, r, "/", http.StatusSeeOther)
		return
	}

	s, err := db.PreviewSpread(id)
	if err != nil {
		log.Printf(`spread: something went wrong with the preview: %s`, err)
		http.Redirect(w, r, "/", http.StatusSeeOther)
		return
	}

	w.Header().Add("Content-Type", "text/plain")
	w.Write([]byte(fmt.Sprintf("Spread of envelope %s looks like this:\n\n", r.FormValue("id"))))
	w.Write([]byte(s))
}

func handleRequest(db *DB, w http.ResponseWriter, r *http.Request) {
	if r.URL.String() != "/" {
		log.Printf(`ignoring request for %s`, r.URL)
		http.NotFound(w, r)
		return
	}

	log.Printf(`request: %v`, r.URL)

	w.Header().Add("Content-Type", "text/html")
	es := db.AllEnvelopes()
	delta := int(0)
	balance := int(0)
	monthtarget := int(0)
	for i := range es {
		delta += es[i].Balance - es[i].Target
		balance += es[i].Balance
		monthtarget += es[i].MonthTarget
	}
	dcls := "delta-ok"
	if delta < 0 {
		dcls = "delta-warn"
	}
	param := struct {
		Envelopes  []*Envelope
		TotalDelta struct {
			Cls string
			Val int
		}
		TotalBalance int
		MonthTarget  int
	}{
		es,
		struct {
			Cls string
			Val int
		}{dcls, delta},
		balance,
		monthtarget,
	}

	if err := templ.ExecuteTemplate(w, "index.html", param); err != nil {
		log.Printf(`error rendering overview template: %s`, err)
	}
}

func main() {
	log.Printf("Here we go")

	db, err := OpenDB()
	if err != nil {
		log.Fatal(err)
	}
	defer func() {
		if err := db.Close(); err != nil {
			log.Printf(`error while saving DB: %s`, err)
		}
	}()

	http.Handle("/static/", http.FileServer(http.Dir(".")))
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		handleRequest(db, w, r)
	})
	http.HandleFunc("/update", func(w http.ResponseWriter, r *http.Request) {
		handleUpdateRequest(db, w, r)
	})
	http.HandleFunc("/delete", func(w http.ResponseWriter, r *http.Request) {
		handleDeleteRequest(db, w, r)
	})
	http.HandleFunc("/details", func(w http.ResponseWriter, r *http.Request) {
		handleDetail(db, w, r)
	})
	http.HandleFunc("/spread", func(w http.ResponseWriter, r *http.Request) {
		handleSpread(db, w, r)
	})
	http.HandleFunc("/tx", func(w http.ResponseWriter, r *http.Request) {
		handleTx(db, w, r)
	})
	http.HandleFunc("/debug", func(w http.ResponseWriter, r *http.Request) {
		handleDebug(w, r)
	})
	err = http.ListenAndServe("127.0.0.1:8081", nil)
	if err != nil {
		log.Printf(`HTTP died: %s`, err)
	}
}
<commit_msg>gofmt<commit_after>package main

import (
	"fmt"
	"html/template"
	"log"
	"net/http"
	"strconv"

	"github.com/google/uuid"
)

var templFuncs = template.FuncMap{
	"prettyDisplay": prettyDisplay,
	"delta":         computeDelta,
}
var templ = template.Must(template.New("").Funcs(templFuncs).ParseGlob("templates/*.html"))

func prettyDisplay(cents int) string {
	return fmt.Sprintf("%.02f", float64(cents)/100)
}

func computeDelta(balance, target int) []string {
	delta := balance - target
	cls := "delta-ok"
	if delta < 0 {
		cls = "delta-warn"
	}
	return []string{cls, fmt.Sprintf(`%.02f`, float64(delta)/100)}
}

func handleDeleteRequest(db *DB, w http.ResponseWriter, r *http.Request) {
	log.Printf(`delete: %v`, r.URL)
	log.Printf(`id: %s`, r.FormValue("id"))

	id, err := uuid.Parse(r.FormValue("id"))
	if err != nil {
		log.Printf(`update: can't parse ID: %s`, err)
		http.Redirect(w, r, "/", http.StatusSeeOther)
		return
	}

	db.DeleteEnvelope(id)

	http.Redirect(w, r, "/", http.StatusSeeOther)
}

func handleUpdateRequest(db *DB, w http.ResponseWriter, r *http.Request) {
	log.Printf(`update: %v`, r.URL)

	log.Printf(`name: %s`, r.FormValue("env-name"))
	log.Printf(`target: %s`, r.FormValue("env-target"))
	log.Printf(`monthtarget: %s`, r.FormValue("env-monthtarget"))
	log.Printf(`return: %s`, r.FormValue("env-return"))

	returnTo := "/"
	if r.FormValue("env-return") != "" {
		returnTo = "/details?id=" + r.FormValue("env-return")
	}

	id, err := uuid.Parse(r.FormValue("env-id"))
	if err != nil {
		log.Printf(`update: can't parse ID: %s`, err)
		id = uuid.New()
	}

	name := r.FormValue("env-name")

	newTarget := 0
	if tgt, err := strconv.ParseFloat(r.FormValue("env-target"), 64); err == nil {
		newTarget = int(tgt * 100)
	}

	newMonthTarget := 0
	if monthtgt, err := strconv.ParseFloat(r.FormValue("env-monthtarget"), 64); err == nil {
		newMonthTarget = int(monthtgt * 100)
	}

	if err = db.UpdateEnvelopeMeta(id, name, newTarget, newMonthTarget); err != nil {
		log.Printf(`can't update envelope %s: %s`, id, err)
		http.Redirect(w, r, returnTo, http.StatusSeeOther)
		return
	}

	http.Redirect(w, r, returnTo, http.StatusSeeOther)
}

func handleDebug(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Content-Type", "text/plain")
	w.Write([]byte("Hic sunt dracones\r\n\r\n"))
}

func handleDetail(db *DB, w http.ResponseWriter, r *http.Request) {
	log.Printf(`handling detail for id %s`, r.FormValue("id"))
	id, err := uuid.Parse(r.FormValue("id"))
	if err != nil {
		log.Printf(`detail: can't parse ID: %s`, err)
		http.Redirect(w, r, "/", http.StatusSeeOther)
		return
	}

	e, events, err := db.EnvelopeWithHistory(id)
	if err != nil {
		log.Printf(`detail: can't get envelope and history from DB: %s`, err)
		http.Redirect(w, r, "/", http.StatusSeeOther)
		return
	}

	events_rev := []Event{}
	for idx := len(events) - 1; idx >= 0; idx-- {
		events_rev = append(events_rev, events[idx])
	}

	param := struct {
		Envelope *Envelope
		Events   []Event
	}{e, events_rev}

	if err := templ.ExecuteTemplate(w, "details.html", param); err != nil {
		log.Printf(`error rendering details template: %s`, err)
	}
}

func handleTx(db *DB, w http.ResponseWriter, r *http.Request) {
	log.Printf(`tx for id %s`, r.FormValue(`id`))

	id, err := uuid.Parse(r.FormValue("id"))
	if err != nil {
		log.Printf(`tx: can't parse ID: %s`, err)
		http.Redirect(w, r, "/", http.StatusSeeOther)
		return
	}

	dir := r.FormValue(`dir`)
	switch dir {
	case `in`:
		/* nothing */
	case `out`:
		/* nothing */
	case `inout`:
		/* nothing */
	default:
		dir = `inout`
	}

	env, err := db.Envelope(id)
	if err != nil {
		log.Printf(`tx: can't get envelope %s: %s`, r.FormValue(`id`), err)
		http.Redirect(w, r, "/", http.StatusSeeOther)
		return
	}

	if r.Method != "POST" {
		switch dir {
		case `in`:
			fallthrough
		case `out`:
			log.Printf(`tx %s`, dir)
			params := struct {
				Envelope  *Envelope
				Direction string
			}{
				Envelope:  env,
				Direction: dir,
			}
			if err := templ.ExecuteTemplate(w, "transfer_in.html", params); err != nil {
				log.Printf(`error rendering details template: %s`, err)
			}
		default:
			params := struct {
				AllEnvelopes []*Envelope
				This         *Envelope
			}{
				AllEnvelopes: []*Envelope{},
				This:         env,
			}
			for _, e := range db.AllEnvelopes() {
				if e.Id != env.Id {
					params.AllEnvelopes = append(params.AllEnvelopes, e)
				}
			}
			if err := templ.ExecuteTemplate(w, "transfer.html", params); err != nil {
				log.Printf(`error rendering details template: %s`, err)
			}
			log.Printf(`tx inout`)
		}
	} else {
		log.Printf(`updating env %s`, r.FormValue(`id`))
		log.Printf(`  amount: %s`, r.FormValue(`amount`))
		amount, err := strconv.ParseFloat(r.FormValue(`amount`), 64)
		if err != nil {
			log.Printf(`can't parse %s: %s`, r.FormValue(`amount`), err)
			http.Redirect(w, r, "/", http.StatusSeeOther)
			return
		}
		switch dir {
		case `in`:
			if err = db.UpdateEnvelopeBalance(id, int(amount*100)); err != nil {
				log.Printf(`can't update balance: %s`, err)
			}
		case `out`:
			if err = db.UpdateEnvelopeBalance(id, -int(amount*100)); err != nil {
				log.Printf(`can't update balance: %s`, err)
			}
		default:
			destId, err := uuid.Parse(r.FormValue("destination"))
			if err != nil {
				log.Printf(`tx: can't parse ID: %s`, err)
				http.Redirect(w, r, "/", http.StatusSeeOther)
				return
			}
			if err = db.UpdateEnvelopeBalance(destId, int(amount*100)); err != nil {
				log.Printf(`can't update balance: %s`, err)
			} else if err = db.UpdateEnvelopeBalance(id, -int(amount*100)); err != nil {
				log.Printf(`can't update balance: %s`, err)
			}
			http.Redirect(w, r, fmt.Sprintf("/details?id=%s", destId), http.StatusSeeOther)
			return
		}
		http.Redirect(w, r, fmt.Sprintf("/details?id=%s", r.FormValue(`id`)), http.StatusSeeOther)
		return
	}
}

func handleSpread(db *DB, w http.ResponseWriter, r *http.Request) {
	log.Printf(`handling spread for id %s`, r.FormValue("id"))

	id, err := uuid.Parse(r.FormValue("id"))
	if err != nil {
		log.Printf(`spread: can't parse ID: %s`, err)
		http.Redirect(w, r, "/", http.StatusSeeOther)
		return
	}

	s, err := db.PreviewSpread(id)
	if err != nil {
		log.Printf(`spread: something went wrong with the preview: %s`, err)
		http.Redirect(w, r, "/", http.StatusSeeOther)
		return
	}

	w.Header().Add("Content-Type", "text/plain")
	w.Write([]byte(fmt.Sprintf("Spread of envelope %s looks like this:\n\n", r.FormValue("id"))))
	w.Write([]byte(s))
}

func handleRequest(db *DB, w http.ResponseWriter, r *http.Request) {
	if r.URL.String() != "/" {
		log.Printf(`ignoring request for %s`, r.URL)
		http.NotFound(w, r)
		return
	}

	log.Printf(`request: %v`, r.URL)

	w.Header().Add("Content-Type", "text/html")
	es := db.AllEnvelopes()
	delta := int(0)
	balance := int(0)
	monthtarget := int(0)
	for i := range es {
		delta += es[i].Balance - es[i].Target
		balance += es[i].Balance
		monthtarget += es[i].MonthTarget
	}
	dcls := "delta-ok"
	if delta < 0 {
		dcls = "delta-warn"
	}
	param := struct {
		Envelopes  []*Envelope
		TotalDelta struct {
			Cls string
			Val int
		}
		TotalBalance int
		MonthTarget  int
	}{
		es,
		struct {
			Cls string
			Val int
		}{dcls, delta},
		balance,
		monthtarget,
	}

	if err := templ.ExecuteTemplate(w, "index.html", param); err != nil {
		log.Printf(`error rendering overview template: %s`, err)
	}
}

func main() {
	log.Printf("Here we go")

	db, err := OpenDB()
	if err != nil {
		log.Fatal(err)
	}
	defer func() {
		if err := db.Close(); err != nil {
			log.Printf(`error while saving DB: %s`, err)
		}
	}()

	http.Handle("/static/", http.FileServer(http.Dir(".")))
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		handleRequest(db, w, r)
	})
	http.HandleFunc("/update", func(w http.ResponseWriter, r *http.Request) {
		handleUpdateRequest(db, w, r)
	})
	http.HandleFunc("/delete", func(w http.ResponseWriter, r *http.Request) {
		handleDeleteRequest(db, w, r)
	})
	http.HandleFunc("/details", func(w http.ResponseWriter, r *http.Request) {
		handleDetail(db, w, r)
	})
	http.HandleFunc("/spread", func(w http.ResponseWriter, r *http.Request) {
		handleSpread(db, w, r)
	})
	http.HandleFunc("/tx", func(w http.ResponseWriter, r *http.Request) {
		handleTx(db, w, r)
	})
	http.HandleFunc("/debug", func(w http.ResponseWriter, r *http.Request) {
		handleDebug(w, r)
	})
	err = http.ListenAndServe("127.0.0.1:8081", nil)
	if err != nil {
		log.Printf(`HTTP died: %s`, err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ebitenutil

import (
	// Import image/png for backward compatibility (#500, #518)
	// TODO: Deprecate and remove this at the next major version (2.0).
	_ "image/png"
)
<commit_msg>ebitenutil: Fix comment (#516)<commit_after>// Copyright 2018 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ebitenutil

import (
	// Import image/png for backward compatibility (#500, #516)
	// TODO: Deprecate and remove this at the next major version (2.0).
	_ "image/png"
)
<|endoftext|>


<commit_before>package consistence

import (
	"errors"
	"strconv"
)

var (
	ErrLeaderSessionAlreadyExist = errors.New("leader session already exist")
)

type NsqdNodeInfo struct {
	ID      string
	NodeIp  string
	TcpPort string
	RpcPort string
}

func (self *NsqdNodeInfo) GetID() string {
	return self.ID
}

type NsqLookupdNodeInfo struct {
	ID       string
	NodeIp   string
	HttpPort string
	TcpPort  string
	RpcPort  string
	Epoch    int
}

func (self *NsqLookupdNodeInfo) GetID() string {
	return self.ID
}

// topicLeaderEpoch increase while leader changed.
// topicEpoch increase while leader or isr changed. (seems not really needed?)
// lookupEpoch increase while lookup leader changed.

type TopicPartionMetaInfo struct {
	Name        string
	Partition   int
	Channels    []string
	Leader      string
	ISR         []string
	CatchupList []string
	Epoch       int
	Replica     int
	// the suggest load factor for each topic partition.
	SuggestLF int
}

func (self *TopicPartionMetaInfo) GetTopicDesp() string {
	return self.Name + "-" + strconv.Itoa(self.Partition)
}

type TopicLeaderSession struct {
	ClusterID   string
	Topic       string
	Partition   int
	LeaderNode  *NsqdNodeInfo
	Session     string
	LeaderEpoch int
}

func (self *TopicLeaderSession) IsSame(other *TopicLeaderSession) bool {
	if other == nil || self == nil {
		return false
	}
	if self.LeaderEpoch != other.LeaderEpoch {
		return false
	}
	if self.Session != "" && other.Session != "" && self.Session != other.Session {
		return false
	}
	if self.LeaderNode == nil || other.LeaderNode == nil {
		return false
	}
	if self.LeaderNode.GetID() != other.LeaderNode.GetID() {
		return false
	}
	return true
}

type ConsistentStore interface {
	WriteKey(key, value string) error
	ReadKey(key string) (string, error)
	ListKey(key string) ([]string, error)
}

// We need check leader lock session before do any modify to etcd.
// Make sure all returned value should be copied to avoid modify by outside.
type NSQLookupdLeadership interface {
	InitClusterID(id string)
	Register(value *NsqLookupdNodeInfo) error
	Unregister() error
	Stop()
	GetAllLookupdNodes() ([]NsqLookupdNodeInfo, error)
	AcquireAndWatchLeader(leader chan *NsqLookupdNodeInfo, stop chan struct{})
	CheckIfLeader(session string) bool
	UpdateLookupEpoch(key string, oldGen int) (int, error)
	WatchNsqdNodes(nsqds chan []*NsqdNodeInfo, stop chan struct{})
	ScanTopics() ([]*TopicPartionMetaInfo, error)
	GetTopicInfo(topic string, partition int) (*TopicPartionMetaInfo, error)
	CreateTopicPartition(topic string, partition int) error
	CreateTopic(topic string, partitionNum int, replica int) error
	IsExistTopic(topic string) (bool, error)
	IsExistTopicPartition(topic string, partitionNum int) (bool, error)
	GetTopicPartitionNum(topic string) (int, error)
	DeleteTopic(topic string, partition int) error
	// update leader, isr, epoch
	// Note: update should do check-and-set to avoid unexpected override.
	CreateTopicNodeInfo(topic string, partition int, topicInfo *TopicPartionMetaInfo) (error, int)
	UpdateTopicNodeInfo(topic string, partition int, topicInfo *TopicPartionMetaInfo, oldGen int) error
	GetTopicLeaderSession(topic string, partition int) (*TopicLeaderSession, error)
	WatchTopicLeader(topic string, partition int, leader chan *TopicLeaderSession, stop chan struct{}) error
}

type NSQDLeadership interface {
	InitClusterID(id string)
	RegisterNsqd(nodeData *NsqdNodeInfo) error
	UnregisterNsqd(nodeData *NsqdNodeInfo) error
	AcquireTopicLeader(topic string, partition int, nodeData *NsqdNodeInfo) error
	ReleaseTopicLeader(topic string, partition int, session *TopicLeaderSession) error
	GetAllLookupdNodes() ([]NsqLookupdNodeInfo, error)
	WatchLookupdLeader(key string, leader chan *NsqLookupdNodeInfo, stop chan struct{}) error
	GetTopicInfo(topic string, partition int) (*TopicPartionMetaInfo, error)
	GetTopicLeaderSession(topic string, partition int) (*TopicLeaderSession, error)
}
<commit_msg>add note for leadership interface<commit_after>package consistence

import (
	"errors"
	"strconv"
)

var (
	ErrLeaderSessionAlreadyExist = errors.New("leader session already exist")
)

type NsqdNodeInfo struct {
	ID      string
	NodeIp  string
	TcpPort string
	RpcPort string
}

func (self *NsqdNodeInfo) GetID() string {
	return self.ID
}

type NsqLookupdNodeInfo struct {
	ID       string
	NodeIp   string
	HttpPort string
	TcpPort  string
	RpcPort  string
	Epoch    int
}

func (self *NsqLookupdNodeInfo) GetID() string {
	return self.ID
}

// topicLeaderEpoch increase while leader changed.
// topicEpoch increase while leader or isr changed. (seems not really needed?)
// lookupEpoch increase while lookup leader changed.

type TopicPartionMetaInfo struct {
	Name        string
	Partition   int
	Channels    []string
	Leader      string
	ISR         []string
	CatchupList []string
	Epoch       int
	Replica     int
	// the suggest load factor for each topic partition.
	SuggestLF int
}

func (self *TopicPartionMetaInfo) GetTopicDesp() string {
	return self.Name + "-" + strconv.Itoa(self.Partition)
}

type TopicLeaderSession struct {
	ClusterID   string
	Topic       string
	Partition   int
	LeaderNode  *NsqdNodeInfo
	Session     string
	LeaderEpoch int
}

func (self *TopicLeaderSession) IsSame(other *TopicLeaderSession) bool {
	if other == nil || self == nil {
		return false
	}
	if self.LeaderEpoch != other.LeaderEpoch {
		return false
	}
	if self.Session != "" && other.Session != "" && self.Session != other.Session {
		return false
	}
	if self.LeaderNode == nil || other.LeaderNode == nil {
		return false
	}
	if self.LeaderNode.GetID() != other.LeaderNode.GetID() {
		return false
	}
	return true
}

type ConsistentStore interface {
	WriteKey(key, value string) error
	ReadKey(key string) (string, error)
	ListKey(key string) ([]string, error)
}

// We need check leader lock session before do any modify to etcd.
// Make sure all returned value should be copied to avoid modify by outside.
type NSQLookupdLeadership interface {
	InitClusterID(id string)
	Register(value *NsqLookupdNodeInfo) error
	Unregister() error
	Stop()
	GetAllLookupdNodes() ([]NsqLookupdNodeInfo, error)
	AcquireAndWatchLeader(leader chan *NsqLookupdNodeInfo, stop chan struct{})
	CheckIfLeader(session string) bool
	UpdateLookupEpoch(key string, oldGen int) (int, error)
	// cluster nsqd node
	WatchNsqdNodes(nsqds chan []NsqdNodeInfo, stop chan struct{})
	ScanTopics() ([]TopicPartionMetaInfo, error)
	GetTopicInfo(topic string, partition int) (*TopicPartionMetaInfo, error)
	CreateTopicPartition(topic string, partition int) error
	CreateTopic(topic string, partitionNum int, replica int) error
	IsExistTopic(topic string) (bool, error)
	IsExistTopicPartition(topic string, partitionNum int) (bool, error)
	GetTopicPartitionNum(topic string) (int, error)
	DeleteTopic(topic string, partition int) error
	// update leader, isr, epoch
	// Note: update should do check-and-set to avoid unexpected override.
	CreateTopicNodeInfo(topic string, partition int, topicInfo *TopicPartionMetaInfo) (error, int)
	UpdateTopicNodeInfo(topic string, partition int, topicInfo *TopicPartionMetaInfo, oldGen int) error
	// get leadership information
	GetTopicLeaderSession(topic string, partition int) (*TopicLeaderSession, error)
	// watch any leadership lock change for all topic partitions, should return the token used later by release.
	WatchTopicLeader(leader chan *TopicLeaderSession, stop chan struct{}) error
}

type NSQDLeadership interface {
	InitClusterID(id string)
	RegisterNsqd(nodeData *NsqdNodeInfo) error
	UnregisterNsqd(nodeData *NsqdNodeInfo) error
	// get the topic leadership lock.
	AcquireTopicLeader(topic string, partition int, nodeData *NsqdNodeInfo) error
	// release the lock using the acquired session.
	ReleaseTopicLeader(topic string, partition int, session *TopicLeaderSession) error
	// all registered lookup nodes.
	GetAllLookupdNodes() ([]NsqLookupdNodeInfo, error)
	WatchLookupdLeader(key string, leader chan *NsqLookupdNodeInfo, stop chan struct{}) error
	GetTopicInfo(topic string, partition int) (*TopicPartionMetaInfo, error)
	GetTopicLeaderSession(topic string, partition int) (*TopicLeaderSession, error)
}
<|endoftext|>