<commit_before>package dbhandler

type credential_struct struct {
    credential_id uint32
    emailaddress string
    password_hash string
}

type address_struct struct {
    address_id uint32
    country string
    zipcode string
    state string
    city string
    line1 string
    line2 string
}

type school_struct struct {
    school_id uint32
    school_name string
    address_id uint32
}

type advisor_struct struct {
    advisor_id uint32
    advisor_name string
    credential_id int32
}

type school_advisor_struct struct {
    advisor_id uint32
    school_id int32
}

type team_struct struct {
    team_id uint32
    team_name string
    team_division string
    advisor_id uint32
}

type student_struct struct {
    student_id uint32
    student_name string
    student_grade string
    school_id uint32
    team_id uint32
}

type team_score_struct struct {
    team_id uint32
    problem_id uint32
}

type parking struct {
    parking_id uint32
    vehicle_count uint32
    validation string
    advisor_id uint32
}

type problem_struct struct {
    problem_id uint32
    round uint32
    division uint32
    problem_desc string
}

type solution_struct struct {
    solution_id uint32
    solution_desc string
}

type problem_solution struct {
    problem_id uint32
    solution_id uint32
}<commit_msg>minor fix<commit_after>package dbhandler

type credential_struct struct {
    credential_id uint32
    emailaddress string
    password_hash string
}

type address_struct struct {
    address_id uint32
    country string
    zipcode string
    state string
    city string
    line1 string
    line2 string
}

type school_struct struct {
    school_id uint32
    school_name string
    address_id uint32
}

type advisor_struct struct {
    advisor_id uint32
    advisor_name string
    credential_id int32
}

type school_advisor_struct struct {
    advisor_id uint32
    school_id int32
}

type team_struct struct {
    team_id uint32
    team_name string
    team_division string
    advisor_id uint32
}

type student_struct struct {
    student_id uint32
    student_name string
    student_grade string
    school_id uint32
    team_id uint32
}

type team_score_struct struct {
    team_id uint32
    problem_id uint32
}

type parking struct {
    parking_id uint32
    vehicle_count uint32
    validation string
    advisor_id uint32
}

type problem_struct struct {
    problem_id uint32
    round uint32
    division uint32
    problem_desc string
}

type solution_struct struct {
    solution_id uint32
    solution_desc string
}

type problem_solution_struct struct {
    problem_id uint32
    solution_id uint32
}<|endoftext|>


<commit_before>package brats_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"github.com/onsi/gomega/gexec"
)

var _ = Describe("Bosh Backup and Restore BBR", func() {
	var backupDir []string

	BeforeEach(func() {
		bbrSdkOps := fmt.Sprintf("-o %s", boshDeploymentAssetPath("bbr.yml"))
		startInnerBosh(bbrSdkOps)
	})

	AfterEach(func() {
		for _, dir := range backupDir {
			err := os.RemoveAll(dir)
			Expect(err).ToNot(HaveOccurred())
		}
		stopInnerBosh()
	})

	Context("database backup", func() {
		It("can backup and restore (removes underlying deployment and release)", func() {
			syslogManifestPath := assetPath("syslog-manifest.yml")
			osConfManifestPath := assetPath("os-conf-manifest.yml")

			By("create syslog deployment", func() {
				uploadStemcell(candidateWardenLinuxStemcellPath)
				uploadRelease("https://bosh.io/d/github.com/cloudfoundry/syslog-release?v=11")

				session := bosh("-n", "deploy", syslogManifestPath,
					"-d", "syslog-deployment",
					"-v", fmt.Sprintf("stemcell-os=%s", stemcellOS),
				)
				Eventually(session, 3*time.Minute).Should(gexec.Exit(0))
			})

			By("create os-conf deployment", func() {
				uploadRelease("https://bosh.io/d/github.com/cloudfoundry/os-conf-release?v=12")

				session := bosh("-n", "deploy", osConfManifestPath,
					"-d", "os-conf-deployment",
					"-v", fmt.Sprintf("stemcell-os=%s", stemcellOS),
				)
				Eventually(session, 3*time.Minute).Should(gexec.Exit(0))
			})

			By("bbr creates a backup", func() {
				session := bbr("director",
					"--host", fmt.Sprintf("%s:22", innerDirectorIP),
					"--username", innerDirectorUser,
					"--private-key-path", innerBoshJumpboxPrivateKeyPath,
					"backup")
				Eventually(session, 2*time.Minute).Should(gexec.Exit(0))
			})

			By("wipe system, recreate inner director", func() {
				stopInnerBosh()

				startInnerBosh()
			})

			By("expect deploy to fail because the release/stemcell won't be there", func() {
				session := bosh("-n", "deploy", syslogManifestPath,
					"-d", "syslog-deployment",
					"-v", fmt.Sprintf("stemcell-os=%s", stemcellOS),
					"--recreate",
				)
				Eventually(session, time.Minute).Should(gexec.Exit(1))
			})

			By("restore inner director from backup", func() {
				var err error
				backupDir, err = filepath.Glob(fmt.Sprintf("%s_*", innerDirectorIP))
				Expect(err).NotTo(HaveOccurred())
				Expect(backupDir).To(HaveLen(1))

				session := bbr("director",
					"--host", fmt.Sprintf("%s:22", innerDirectorIP),
					"--username", innerDirectorUser,
					"--private-key-path", innerBoshJumpboxPrivateKeyPath,
					"restore",
					"--artifact-path", backupDir[0])
				Eventually(session, 2*time.Minute).Should(gexec.Exit(0))

				waitForBoshDirectorUp(boshBinaryPath)

				session = bosh("-n", "upload-stemcell", "--fix", candidateWardenLinuxStemcellPath)
				Eventually(session, 5*time.Minute).Should(gexec.Exit(0))
			})

			By("cck the deployments", func() {
				session := bosh("-n", "-d", "syslog-deployment", "cck", "--auto")
				Eventually(session, 10*time.Minute).Should(gexec.Exit(0))

				session = bosh("-n", "-d", "os-conf-deployment", "cck",
					"--resolution", "delete_vm_reference",
					"--resolution", "delete_disk_reference")
				Eventually(session, 10*time.Minute).Should(gexec.Exit(0))

				session = bosh("-n", "deploy", osConfManifestPath,
					"-d", "os-conf-deployment",
					"-v", fmt.Sprintf("stemcell-os=%s", stemcellOS),
				)
				Eventually(session, 3*time.Minute).Should(gexec.Exit(0))
			})

			By("validate deployments", func() {
				By("instance actually ran the jobs", func() {
					session := bosh("-n", "-d", "syslog-deployment", "instances",
						"--ps",
						"--column=process_state",
						"--column=instance")
					Eventually(session, time.Minute).Should(gexec.Exit(0))

					Expect(session.Out.Contents()).To(MatchRegexp("syslog_storer/[a-z0-9-]+[ \t]+running"))
					Expect(session.Out.Contents()).To(MatchRegexp("syslog_forwarder/[a-z0-9-]+[ \t]+running"))
				})

				By("persistent disks exist", func() {
					session := bosh("-n", "-d", "os-conf-deployment", "instances",
						"--details",
						"--column", "disk_cids",
					)

					Eventually(session, time.Minute).Should(gexec.Exit(0))

					Expect(string(session.Out.Contents())).To(MatchRegexp("[0-9a-f]{8}-[0-9a-f-]{27}"))
				})
			})
		})

		It("can backup and restore (reattaches to underlying deployment)", func() {
			By("Set up a deployment that uses the syslog release", func() {
				uploadStemcell(candidateWardenLinuxStemcellPath)
				uploadRelease("https://bosh.io/d/github.com/cloudfoundry/syslog-release?v=11")

				manifestPath := assetPath("syslog-manifest.yml")
				session := bosh("-n", "deploy", manifestPath,
					"-d", "syslog-deployment",
					"-v", fmt.Sprintf("stemcell-os=%s", stemcellOS),
				)

				Eventually(session, 3*time.Minute).Should(gexec.Exit(0))
			})

			By("bbr creates a backup", func() {
				session := bbr("director",
					"--host", fmt.Sprintf("%s:22", innerDirectorIP),
					"--username", innerDirectorUser,
					"--private-key-path", innerBoshJumpboxPrivateKeyPath,
					"backup")
				Eventually(session, time.Minute).Should(gexec.Exit(0))
			})

			By("wipe system", func() {
				session := outerBosh("-d", "bosh",
					"ssh", "bosh", "sudo rm -rf /var/vcap/store/blobstore/*")
				Eventually(session, 5*time.Minute).Should(gexec.Exit(0))

				var err error
				backupDir, err = filepath.Glob(fmt.Sprintf("%s_*", innerDirectorIP))
				Expect(err).NotTo(HaveOccurred())
				Expect(backupDir).To(HaveLen(1))

				session = bbr("director",
					"--host", fmt.Sprintf("%s:22", innerDirectorIP),
					"--username", innerDirectorUser,
					"--private-key-path", innerBoshJumpboxPrivateKeyPath,
					"restore",
					"--artifact-path", backupDir[0])
				Eventually(session, time.Minute).Should(gexec.Exit(0))

				waitForBoshDirectorUp(boshBinaryPath)

				session = bosh("-n", "upload-stemcell", "--fix", candidateWardenLinuxStemcellPath)
				Eventually(session, 5*time.Minute).Should(gexec.Exit(0))

				session = bosh("-n", "-d", "syslog-deployment", "cck", "--report")
				Eventually(session, 3*time.Minute).Should(gexec.Exit(0))
			})

			By("validate deployment. instance actually ran the jobs", func() {
				session := bosh("-n", "-d", "syslog-deployment", "instances",
					"--ps",
					"--column=process_state",
					"--column=instance",
				)
				Eventually(session, time.Minute).Should(gexec.Exit(0))

				Expect(session.Out.Contents()).To(MatchRegexp("syslog_storer/[a-z0-9-]+[ \t]+running"))
				Expect(session.Out.Contents()).To(MatchRegexp("syslog_forwarder/[a-z0-9-]+[ \t]+running"))
			})
		})
	})

	Context("blobstore files", func() {
		var directoriesBefore, filesBefore []string

		It("backs up an empty blobstore", func() {
			By("Backup deployment", func() {
				session := bbr("director",
					"--host", fmt.Sprintf("%s:22", innerDirectorIP),
					"--username", innerDirectorUser,
					"--private-key-path", innerBoshJumpboxPrivateKeyPath,
					"backup")
				Eventually(session, time.Minute).Should(gexec.Exit(0))
			})

			By("Restore deployment", func() {
				var err error
				backupDir, err = filepath.Glob(fmt.Sprintf("%s_*", innerDirectorIP))
				Expect(err).NotTo(HaveOccurred())
				Expect(backupDir).To(HaveLen(1))

				session := bbr("director",
					"--host", fmt.Sprintf("%s:22", innerDirectorIP),
					"--username", innerDirectorUser,
					"--private-key-path", innerBoshJumpboxPrivateKeyPath,
					"restore",
					"--artifact-path", backupDir[0])
				Eventually(session, time.Minute).Should(gexec.Exit(0))

				waitForBoshDirectorUp(boshBinaryPath)
			})
		})

		It("restores the blobstore files with the correct permissions/ownership", func() {
			By("Upload a release", func() {
				uploadRelease("https://bosh.io/d/github.com/cloudfoundry/syslog-release?v=11")
			})

			By("Store directory/file structure before we do backup", func() {
				directoriesBefore, filesBefore = findBlobstoreFiles(outerBoshBinaryPath)
			})

			By("Backup deployment", func() {
				session := bbr("director",
					"--host", fmt.Sprintf("%s:22", innerDirectorIP),
					"--username", innerDirectorUser,
					"--private-key-path", innerBoshJumpboxPrivateKeyPath,
					"backup")
				Eventually(session, time.Minute).Should(gexec.Exit(0))
			})

			By("Check directories are still there after backup", func() {
				directoriesAfter, filesAfter := findBlobstoreFiles(outerBoshBinaryPath)
				Expect(directoriesAfter).To(Equal(directoriesBefore))
				Expect(filesAfter).To(Equal(filesBefore))
			})

			By("\"wipe\" system", func() {
				session := outerBosh("-d", "bosh", "ssh", "bosh", "-c", "sudo rm -rf /var/vcap/store/blobstore/*")
				Eventually(session, 5*time.Minute).Should(gexec.Exit(0))
			})

			By("Restore deployment", func() {
				var err error
				backupDir, err = filepath.Glob(fmt.Sprintf("%s_*", innerDirectorIP))
				Expect(err).NotTo(HaveOccurred())
				Expect(backupDir).To(HaveLen(1))

				session := bbr("director",
					"--host", fmt.Sprintf("%s:22", innerDirectorIP),
					"--username", innerDirectorUser,
					"--private-key-path", innerBoshJumpboxPrivateKeyPath,
					"restore",
					"--artifact-path", backupDir[0])
				Eventually(session, time.Minute).Should(gexec.Exit(0))

				waitForBoshDirectorUp(boshBinaryPath)
			})

			By("Check directories have correct permissions after restore", func() {
				directoriesAfter, filesAfter := findBlobstoreFiles(outerBoshBinaryPath)
				Expect(directoriesAfter).To(Equal(directoriesBefore))
				Expect(filesAfter).To(Equal(filesBefore))
			})
		})
	})
})

func waitForBoshDirectorUp(boshBinaryPath string) {
	Eventually(func() *gexec.Session {
		session := bosh("env")
		session.Wait()
		return session
	}, 5*time.Minute, 2*time.Second).Should(gexec.Exit(0))
}

func findBlobstoreFiles(outerBoshBinaryPath string) ([]string, []string) {
	session := outerBosh("-d", "bosh", "ssh", "bosh", "--results", "--column", "stdout",
		"sudo find /var/vcap/store/blobstore/store -type d -perm 0700 -user vcap -group vcap")
	Eventually(session, time.Minute).Should(gexec.Exit(0))

	directories := strings.Split(string(session.Out.Contents()), "\n")
	sort.Strings(directories)

	session = outerBosh("-d", "bosh", "ssh", "bosh", "-r", "--column", "stdout",
		"sudo find /var/vcap/store/blobstore/store -type f -perm 0600 -user vcap -group vcap")
	Eventually(session, time.Minute).Should(gexec.Exit(0))

	files := strings.Split(string(session.Out.Contents()), "\n")
	sort.Strings(files)

	return directories, files
}
<commit_msg>Fix failing BBR brats test<commit_after>package brats_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"github.com/onsi/gomega/gexec"
)

var _ = Describe("Bosh Backup and Restore BBR", func() {
	var (
		backupDir []string
		bbrSdkOps string
	)

	BeforeEach(func() {
		bbrSdkOps = fmt.Sprintf("-o %s", boshDeploymentAssetPath("bbr.yml"))
		startInnerBosh(bbrSdkOps)
	})

	AfterEach(func() {
		for _, dir := range backupDir {
			err := os.RemoveAll(dir)
			Expect(err).ToNot(HaveOccurred())
		}
		stopInnerBosh()
	})

	Context("database backup", func() {
		It("can backup and restore (removes underlying deployment and release)", func() {
			syslogManifestPath := assetPath("syslog-manifest.yml")
			osConfManifestPath := assetPath("os-conf-manifest.yml")

			By("create syslog deployment", func() {
				uploadStemcell(candidateWardenLinuxStemcellPath)
				uploadRelease("https://bosh.io/d/github.com/cloudfoundry/syslog-release?v=11")

				session := bosh("-n", "deploy", syslogManifestPath,
					"-d", "syslog-deployment",
					"-v", fmt.Sprintf("stemcell-os=%s", stemcellOS),
				)
				Eventually(session, 3*time.Minute).Should(gexec.Exit(0))
			})

			By("create os-conf deployment", func() {
				uploadRelease("https://bosh.io/d/github.com/cloudfoundry/os-conf-release?v=12")

				session := bosh("-n", "deploy", osConfManifestPath,
					"-d", "os-conf-deployment",
					"-v", fmt.Sprintf("stemcell-os=%s", stemcellOS),
				)
				Eventually(session, 3*time.Minute).Should(gexec.Exit(0))
			})

			By("bbr creates a backup", func() {
				session := bbr("director",
					"--host", fmt.Sprintf("%s:22", innerDirectorIP),
					"--username", innerDirectorUser,
					"--private-key-path", innerBoshJumpboxPrivateKeyPath,
					"backup")
				Eventually(session, 2*time.Minute).Should(gexec.Exit(0))
			})

			By("wipe system, recreate inner director", func() {
				stopInnerBosh()
				startInnerBosh(bbrSdkOps)
			})

			By("expect deploy to fail because the release/stemcell won't be there", func() {
				session := bosh("-n", "deploy", syslogManifestPath,
					"-d", "syslog-deployment",
					"-v", fmt.Sprintf("stemcell-os=%s", stemcellOS),
					"--recreate",
				)
				Eventually(session, time.Minute).Should(gexec.Exit(1))
			})

			By("restore inner director from backup", func() {
				var err error
				backupDir, err = filepath.Glob(fmt.Sprintf("%s_*", innerDirectorIP))
				Expect(err).NotTo(HaveOccurred())
				Expect(backupDir).To(HaveLen(1))

				session := bbr("director",
					"--host", fmt.Sprintf("%s:22", innerDirectorIP),
					"--username", innerDirectorUser,
					"--private-key-path", innerBoshJumpboxPrivateKeyPath,
					"restore",
					"--artifact-path", backupDir[0])
				Eventually(session, 2*time.Minute).Should(gexec.Exit(0))

				waitForBoshDirectorUp(boshBinaryPath)

				session = bosh("-n", "upload-stemcell", "--fix", candidateWardenLinuxStemcellPath)
				Eventually(session, 5*time.Minute).Should(gexec.Exit(0))
			})

			By("cck the deployments", func() {
				session := bosh("-n", "-d", "syslog-deployment", "cck", "--auto")
				Eventually(session, 10*time.Minute).Should(gexec.Exit(0))

				session = bosh("-n", "-d", "os-conf-deployment", "cck",
					"--resolution", "delete_vm_reference",
					"--resolution", "delete_disk_reference")
				Eventually(session, 10*time.Minute).Should(gexec.Exit(0))

				session = bosh("-n", "deploy", osConfManifestPath,
					"-d", "os-conf-deployment",
					"-v", fmt.Sprintf("stemcell-os=%s", stemcellOS),
				)
				Eventually(session, 3*time.Minute).Should(gexec.Exit(0))
			})

			By("validate deployments", func() {
				By("instance actually ran the jobs", func() {
					session := bosh("-n", "-d", "syslog-deployment", "instances",
						"--ps",
						"--column=process_state",
						"--column=instance")
					Eventually(session, time.Minute).Should(gexec.Exit(0))

					Expect(session.Out.Contents()).To(MatchRegexp("syslog_storer/[a-z0-9-]+[ \t]+running"))
					Expect(session.Out.Contents()).To(MatchRegexp("syslog_forwarder/[a-z0-9-]+[ \t]+running"))
				})

				By("persistent disks exist", func() {
					session := bosh("-n", "-d", "os-conf-deployment", "instances",
						"--details",
						"--column", "disk_cids",
					)

					Eventually(session, time.Minute).Should(gexec.Exit(0))

					Expect(string(session.Out.Contents())).To(MatchRegexp("[0-9a-f]{8}-[0-9a-f-]{27}"))
				})
			})
		})

		It("can backup and restore (reattaches to underlying deployment)", func() {
			By("Set up a deployment that uses the syslog release", func() {
				uploadStemcell(candidateWardenLinuxStemcellPath)
				uploadRelease("https://bosh.io/d/github.com/cloudfoundry/syslog-release?v=11")

				manifestPath := assetPath("syslog-manifest.yml")
				session := bosh("-n", "deploy", manifestPath,
					"-d", "syslog-deployment",
					"-v", fmt.Sprintf("stemcell-os=%s", stemcellOS),
				)

				Eventually(session, 3*time.Minute).Should(gexec.Exit(0))
			})

			By("bbr creates a backup", func() {
				session := bbr("director",
					"--host", fmt.Sprintf("%s:22", innerDirectorIP),
					"--username", innerDirectorUser,
					"--private-key-path", innerBoshJumpboxPrivateKeyPath,
					"backup")
				Eventually(session, time.Minute).Should(gexec.Exit(0))
			})

			By("wipe system", func() {
				session := outerBosh("-d", "bosh",
					"ssh", "bosh", "sudo rm -rf /var/vcap/store/blobstore/*")
				Eventually(session, 5*time.Minute).Should(gexec.Exit(0))

				var err error
				backupDir, err = filepath.Glob(fmt.Sprintf("%s_*", innerDirectorIP))
				Expect(err).NotTo(HaveOccurred())
				Expect(backupDir).To(HaveLen(1))

				session = bbr("director",
					"--host", fmt.Sprintf("%s:22", innerDirectorIP),
					"--username", innerDirectorUser,
					"--private-key-path", innerBoshJumpboxPrivateKeyPath,
					"restore",
					"--artifact-path", backupDir[0])
				Eventually(session, time.Minute).Should(gexec.Exit(0))

				waitForBoshDirectorUp(boshBinaryPath)

				session = bosh("-n", "upload-stemcell", "--fix", candidateWardenLinuxStemcellPath)
				Eventually(session, 5*time.Minute).Should(gexec.Exit(0))

				session = bosh("-n", "-d", "syslog-deployment", "cck", "--report")
				Eventually(session, 3*time.Minute).Should(gexec.Exit(0))
			})

			By("validate deployment. instance actually ran the jobs", func() {
				session := bosh("-n", "-d", "syslog-deployment", "instances",
					"--ps",
					"--column=process_state",
					"--column=instance",
				)
				Eventually(session, time.Minute).Should(gexec.Exit(0))

				Expect(session.Out.Contents()).To(MatchRegexp("syslog_storer/[a-z0-9-]+[ \t]+running"))
				Expect(session.Out.Contents()).To(MatchRegexp("syslog_forwarder/[a-z0-9-]+[ \t]+running"))
			})
		})
	})

	Context("blobstore files", func() {
		var directoriesBefore, filesBefore []string

		It("backs up an empty blobstore", func() {
			By("Backup deployment", func() {
				session := bbr("director",
					"--host", fmt.Sprintf("%s:22", innerDirectorIP),
					"--username", innerDirectorUser,
					"--private-key-path", innerBoshJumpboxPrivateKeyPath,
					"backup")
				Eventually(session, time.Minute).Should(gexec.Exit(0))
			})

			By("Restore deployment", func() {
				var err error
				backupDir, err = filepath.Glob(fmt.Sprintf("%s_*", innerDirectorIP))
				Expect(err).NotTo(HaveOccurred())
				Expect(backupDir).To(HaveLen(1))

				session := bbr("director",
					"--host", fmt.Sprintf("%s:22", innerDirectorIP),
					"--username", innerDirectorUser,
					"--private-key-path", innerBoshJumpboxPrivateKeyPath,
					"restore",
					"--artifact-path", backupDir[0])
				Eventually(session, time.Minute).Should(gexec.Exit(0))

				waitForBoshDirectorUp(boshBinaryPath)
			})
		})

		It("restores the blobstore files with the correct permissions/ownership", func() {
			By("Upload a release", func() {
				uploadRelease("https://bosh.io/d/github.com/cloudfoundry/syslog-release?v=11")
			})

			By("Store directory/file structure before we do backup", func() {
				directoriesBefore, filesBefore = findBlobstoreFiles(outerBoshBinaryPath)
			})

			By("Backup deployment", func() {
				session := bbr("director",
					"--host", fmt.Sprintf("%s:22", innerDirectorIP),
					"--username", innerDirectorUser,
					"--private-key-path", innerBoshJumpboxPrivateKeyPath,
					"backup")
				Eventually(session, time.Minute).Should(gexec.Exit(0))
			})

			By("Check directories are still there after backup", func() {
				directoriesAfter, filesAfter := findBlobstoreFiles(outerBoshBinaryPath)
				Expect(directoriesAfter).To(Equal(directoriesBefore))
				Expect(filesAfter).To(Equal(filesBefore))
			})

			By("\"wipe\" system", func() {
				session := outerBosh("-d", "bosh", "ssh", "bosh", "-c", "sudo rm -rf /var/vcap/store/blobstore/*")
				Eventually(session, 5*time.Minute).Should(gexec.Exit(0))
			})

			By("Restore deployment", func() {
				var err error
				backupDir, err = filepath.Glob(fmt.Sprintf("%s_*", innerDirectorIP))
				Expect(err).NotTo(HaveOccurred())
				Expect(backupDir).To(HaveLen(1))

				session := bbr("director",
					"--host", fmt.Sprintf("%s:22", innerDirectorIP),
					"--username", innerDirectorUser,
					"--private-key-path", innerBoshJumpboxPrivateKeyPath,
					"restore",
					"--artifact-path", backupDir[0])
				Eventually(session, time.Minute).Should(gexec.Exit(0))

				waitForBoshDirectorUp(boshBinaryPath)
			})

			By("Check directories have correct permissions after restore", func() {
				directoriesAfter, filesAfter := findBlobstoreFiles(outerBoshBinaryPath)
				Expect(directoriesAfter).To(Equal(directoriesBefore))
				Expect(filesAfter).To(Equal(filesBefore))
			})
		})
	})
})

func waitForBoshDirectorUp(boshBinaryPath string) {
	Eventually(func() *gexec.Session {
		session := bosh("env")
		session.Wait()
		return session
	}, 5*time.Minute, 2*time.Second).Should(gexec.Exit(0))
}

func findBlobstoreFiles(outerBoshBinaryPath string) ([]string, []string) {
	session := outerBosh("-d", "bosh", "ssh", "bosh", "--results", "--column", "stdout",
		"sudo find /var/vcap/store/blobstore/store -type d -perm 0700 -user vcap -group vcap")
	Eventually(session, time.Minute).Should(gexec.Exit(0))

	directories := strings.Split(string(session.Out.Contents()), "\n")
	sort.Strings(directories)

	session = outerBosh("-d", "bosh", "ssh", "bosh", "-r", "--column", "stdout",
		"sudo find /var/vcap/store/blobstore/store -type f -perm 0600 -user vcap -group vcap")
	Eventually(session, time.Minute).Should(gexec.Exit(0))

	files := strings.Split(string(session.Out.Contents()), "\n")
	sort.Strings(files)

	return directories, files
}
<|endoftext|>


<commit_before>package aggregator

import (
	"encoding/json"
	"log"
	"sync"
)

const (
	DELIMITER = byte(10)
)

type Publisher interface {
	Publish(b []byte)
}

type Aggregator interface {
	Sink(payload interface{})
	Start()
	Shutdown()
}

type aggregator struct {
	wg         sync.WaitGroup
	channel    chan interface{}
	publisher  Publisher
	bufferSize int
	numWorkers int
}

func NewAggregator(bufferSize int, numWorkers int, publisher Publisher) *aggregator {
	return &aggregator{
		wg:         sync.WaitGroup{},
		channel:    make(chan interface{}, 1000),
		publisher:  publisher,
		bufferSize: bufferSize,
		numWorkers: numWorkers,
	}
}

func (t *aggregator) Start() {
	for i := 0; i < t.numWorkers; i++ {
		go t.worker()
	}
}

func (t *aggregator) Shutdown() {
	close(t.channel)
	t.wg.Wait()
	log.Println("Aggregator Shutdown")
}

func (t *aggregator) Sink(data interface{}) {
	t.channel <- data
}

func (t *aggregator) worker() {
	t.wg.Add(1)
	defer t.wg.Done()

	buffer := make([]byte, t.bufferSize)
	counter := 0

	for payload := range t.channel {
		b, err := json.Marshal(payload)
		if err != nil {
			log.Println("unable to jsonify payload")
			continue
		}
		payloadSize := len(b) + 1

		if counter+payloadSize >= t.bufferSize {
			log.Printf("Fill rate %d/%d=%.02f\n", counter, t.bufferSize, float32(counter)/float32(t.bufferSize))
			go t.publisher.Publish(buffer)
			buffer = make([]byte, t.bufferSize)
			counter = 0
		}

		for i := 0; i < payloadSize-1; i++ {
			buffer[counter] = b[i]
			counter++
		}
		buffer[counter] = DELIMITER
		counter++
	}

	if counter > 0 {
		t.publisher.Publish(buffer)
	}
}

/*

		if t.publisher != nil {
			t.publisher.Publish(b)
		}

	if len(t.channel) == t.batchSize {
		close(t.channel)

		t.aggregate(t.channel)
		t.channel = make(chan Payload, t.batchSize)
	}
}

func (t *aggregator) aggregate(channel chan Payload) {
	t.wg.Add(1)
	defer t.wg.Done()

	for payload := range channel {

		if t.publisher != nil {
			t.publisher.Publish(b)
		}
	}
}
*/
<commit_msg>updating aggregator to have a timeout function<commit_after>package aggregator

import (
	"encoding/json"
	"log"
	"sync"
	"time"
)

const (
	DELIMITER = byte(10)
)

type Publisher interface {
	Publish(b []byte)
}

type Aggregator interface {
	Sink(payload interface{})
	Start()
	Shutdown()
}

type aggregator struct {
	wg         sync.WaitGroup
	channel    chan interface{}
	publisher  Publisher
	bufferSize int
	numWorkers int
	timeout    time.Duration
}

func NewAggregator(bufferSize int, timeout time.Duration, numWorkers int, publisher Publisher) *aggregator {
	return &aggregator{
		wg:         sync.WaitGroup{},
		channel:    make(chan interface{}, 1000),
		publisher:  publisher,
		bufferSize: bufferSize,
		numWorkers: numWorkers,
		timeout:    timeout,
	}
}

func (t *aggregator) Start() {
	for i := 0; i < t.numWorkers; i++ {
		go t.worker()
	}
}

func (t *aggregator) Shutdown() {
	close(t.channel)
	t.wg.Wait()
	log.Println("Aggregator Shutdown")
}

func (t *aggregator) Sink(data interface{}) {
	t.channel <- data
}

func (t *aggregator) worker() {
	t.wg.Add(1)
	defer t.wg.Done()

	buffer := make([]byte, t.bufferSize)
	counter := 0

	timeoutChannel := time.After(t.timeout)
	for {
		select {
		case payload, ok := <-t.channel:
			if !ok {
				log.Println("Aggregator: channel closed")
				if counter > 0 {
					t.publisher.Publish(buffer[:counter])
				}
				break
			}

			log.Println("Aggregator: received message")
			b, err := json.Marshal(payload)
			if err != nil {
				log.Println("unable to jsonify payload")
				continue
			}
			payloadSize := len(b) + 1

			if counter+payloadSize >= t.bufferSize {
				log.Printf("Fill rate %d/%d=%.02f\n", counter, t.bufferSize, float32(counter)/float32(t.bufferSize))
				t.publisher.Publish(buffer[:counter])
				timeoutChannel = time.After(t.timeout)
				counter = 0
			}

			for i := 0; i < payloadSize-1; i++ {
				buffer[counter] = b[i]
				counter++
			}
			buffer[counter] = DELIMITER
			counter++

		case <-timeoutChannel:
			log.Println("Aggregator: timeout")
			if counter > 0 {
				t.publisher.Publish(buffer[:counter])
				timeoutChannel = time.After(t.timeout)
				counter = 0
			}
		}
	}
}

/*

		if t.publisher != nil {
			t.publisher.Publish(b)
		}

	if len(t.channel) == t.batchSize {
		close(t.channel)

		t.aggregate(t.channel)
		t.channel = make(chan Payload, t.batchSize)
	}
}

func (t *aggregator) aggregate(channel chan Payload) {
	t.wg.Add(1)
	defer t.wg.Done()

	for payload := range channel {

		if t.publisher != nil {
			t.publisher.Publish(b)
		}
	}
}
*/
<|endoftext|>


<commit_before>package planmodifiers

import (
	"context"

	"github.com/hashicorp/terraform-plugin-framework/attr"
	"github.com/hashicorp/terraform-plugin-framework/path"
	"github.com/hashicorp/terraform-plugin-framework/tfsdk"
	"github.com/hashicorp/terraform-plugin-framework/types"
)

// DefaultValue accepts an attr.Value and uses the supplied value to set a default if the config for
// the attribute is null.
func DefaultValue(val attr.Value) tfsdk.AttributePlanModifier {
	return &defaultValueAttributePlanModifier{val}
}

type defaultValueAttributePlanModifier struct {
	val attr.Value
}

func (d *defaultValueAttributePlanModifier) Description(ctx context.Context) string {
	return "If the config does not contain a value, a default will be set using val."
}

func (d *defaultValueAttributePlanModifier) MarkdownDescription(ctx context.Context) string {
	return d.Description(ctx)
}

// Modify checks that the value of the attribute in the configuration and assigns the default value if
// the value in the config is null. This is a destructive operation in that it will overwrite any value
// present in the plan.
func (d *defaultValueAttributePlanModifier) Modify(ctx context.Context, req tfsdk.ModifyAttributePlanRequest, resp *tfsdk.ModifyAttributePlanResponse) {
	// Do not set default if the attribute configuration has been set.
	if !req.AttributeConfig.IsNull() {
		return
	}

	resp.AttributePlan = d.val
}

// RequiresReplace returns an attribute plan modifier that is identical to resource.RequiresReplace() with
// the exception that there is no check for `configRaw.IsNull && attrSchema.Computed` as a replacement
// needs to be triggered when the attribute has been removed from the config.
func RequiresReplace() tfsdk.AttributePlanModifier {
	return RequiresReplaceModifier{}
}

type RequiresReplaceModifier struct{}

// Description returns a human-readable description of the plan modifier.
func (r RequiresReplaceModifier) Description(ctx context.Context) string {
	return "If the value of this attribute changes, Terraform will destroy and recreate the resource."
}

// MarkdownDescription returns a markdown description of the plan modifier.
func (r RequiresReplaceModifier) MarkdownDescription(ctx context.Context) string {
	return r.Description(ctx)
}

// Modify will trigger replacement (i.e., destroy-create) when `configRaw.IsNull && attrSchema.Computed`,
// which differs from the behaviour of `resource.RequiresReplace()`.
func (r RequiresReplaceModifier) Modify(ctx context.Context, req tfsdk.ModifyAttributePlanRequest, resp *tfsdk.ModifyAttributePlanResponse) {
	if req.AttributeConfig == nil || req.AttributePlan == nil || req.AttributeState == nil {
		// shouldn't happen, but let's not panic if it does
		return
	}

	if req.State.Raw.IsNull() {
		// if we're creating the resource, no need to delete and
		// recreate it
		return
	}

	if req.Plan.Raw.IsNull() {
		// if we're deleting the resource, no need to delete and
		// recreate it
		return
	}

	if req.AttributePlan.Equal(req.AttributeState) {
		// if the plan and the state are in agreement, this attribute
		// isn't changing, don't require replace
		return
	}

	resp.RequiresReplace = true
}

func NumberNumericAttributePlanModifier() tfsdk.AttributePlanModifier {
	return &numberNumericAttributePlanModifier{}
}

type numberNumericAttributePlanModifier struct {
}

func (d *numberNumericAttributePlanModifier) Description(ctx context.Context) string {
	return "Ensures that number and numeric attributes are kept synchronised."
}

func (d *numberNumericAttributePlanModifier) MarkdownDescription(ctx context.Context) string {
	return d.Description(ctx)
}

func (d *numberNumericAttributePlanModifier) Modify(ctx context.Context, req tfsdk.ModifyAttributePlanRequest, resp *tfsdk.ModifyAttributePlanResponse) {
	numberConfig := types.Bool{}
	diags := req.Config.GetAttribute(ctx, path.Root("number"), &numberConfig)
	resp.Diagnostics.Append(diags...)
	if resp.Diagnostics.HasError() {
		return
	}

	numericConfig := types.Bool{}
	req.Config.GetAttribute(ctx, path.Root("numeric"), &numericConfig)
	resp.Diagnostics.Append(diags...)
	if resp.Diagnostics.HasError() {
		return
	}

	if !numberConfig.Null && !numericConfig.Null {
		resp.Diagnostics.AddError(
			"Number numeric attribute plan modifier failed",
			"Cannot specify both number and numeric in config",
		)
		return
	}

	// Default to true for both number and numeric when both are null.
	if numberConfig.Null && numericConfig.Null {
		resp.AttributePlan = types.Bool{Value: true}
		return
	}

	// Default to using value for numeric if number is null
	if numberConfig.Null && !numericConfig.Null {
		resp.AttributePlan = numericConfig
		return
	}

	// Default to using value for number if numeric is null
	if !numberConfig.Null && numericConfig.Null {
		resp.AttributePlan = numberConfig
		return
	}
}
<commit_msg>Checking diags on GetAttribute() call (#296)<commit_after>package planmodifiers

import (
	"context"

	"github.com/hashicorp/terraform-plugin-framework/attr"
	"github.com/hashicorp/terraform-plugin-framework/path"
	"github.com/hashicorp/terraform-plugin-framework/tfsdk"
	"github.com/hashicorp/terraform-plugin-framework/types"
)

// DefaultValue accepts an attr.Value and uses the supplied value to set a default if the config for
// the attribute is null.
func DefaultValue(val attr.Value) tfsdk.AttributePlanModifier {
	return &defaultValueAttributePlanModifier{val}
}

type defaultValueAttributePlanModifier struct {
	val attr.Value
}

func (d *defaultValueAttributePlanModifier) Description(ctx context.Context) string {
	return "If the config does not contain a value, a default will be set using val."
}

func (d *defaultValueAttributePlanModifier) MarkdownDescription(ctx context.Context) string {
	return d.Description(ctx)
}

// Modify checks that the value of the attribute in the configuration and assigns the default value if
// the value in the config is null. This is a destructive operation in that it will overwrite any value
// present in the plan.
func (d *defaultValueAttributePlanModifier) Modify(ctx context.Context, req tfsdk.ModifyAttributePlanRequest, resp *tfsdk.ModifyAttributePlanResponse) {
	// Do not set default if the attribute configuration has been set.
	if !req.AttributeConfig.IsNull() {
		return
	}

	resp.AttributePlan = d.val
}

// RequiresReplace returns an attribute plan modifier that is identical to resource.RequiresReplace() with
// the exception that there is no check for `configRaw.IsNull && attrSchema.Computed` as a replacement
// needs to be triggered when the attribute has been removed from the config.
func RequiresReplace() tfsdk.AttributePlanModifier {
	return RequiresReplaceModifier{}
}

type RequiresReplaceModifier struct{}

// Description returns a human-readable description of the plan modifier.
func (r RequiresReplaceModifier) Description(ctx context.Context) string {
	return "If the value of this attribute changes, Terraform will destroy and recreate the resource."
}

// MarkdownDescription returns a markdown description of the plan modifier.
func (r RequiresReplaceModifier) MarkdownDescription(ctx context.Context) string {
	return r.Description(ctx)
}

// Modify will trigger replacement (i.e., destroy-create) when `configRaw.IsNull && attrSchema.Computed`,
// which differs from the behaviour of `resource.RequiresReplace()`.
func (r RequiresReplaceModifier) Modify(ctx context.Context, req tfsdk.ModifyAttributePlanRequest, resp *tfsdk.ModifyAttributePlanResponse) {
	if req.AttributeConfig == nil || req.AttributePlan == nil || req.AttributeState == nil {
		// shouldn't happen, but let's not panic if it does
		return
	}

	if req.State.Raw.IsNull() {
		// if we're creating the resource, no need to delete and
		// recreate it
		return
	}

	if req.Plan.Raw.IsNull() {
		// if we're deleting the resource, no need to delete and
		// recreate it
		return
	}

	if req.AttributePlan.Equal(req.AttributeState) {
		// if the plan and the state are in agreement, this attribute
		// isn't changing, don't require replace
		return
	}

	resp.RequiresReplace = true
}

func NumberNumericAttributePlanModifier() tfsdk.AttributePlanModifier {
	return &numberNumericAttributePlanModifier{}
}

type numberNumericAttributePlanModifier struct {
}

func (d *numberNumericAttributePlanModifier) Description(ctx context.Context) string {
	return "Ensures that number and numeric attributes are kept synchronised."
}

func (d *numberNumericAttributePlanModifier) MarkdownDescription(ctx context.Context) string {
	return d.Description(ctx)
}

func (d *numberNumericAttributePlanModifier) Modify(ctx context.Context, req tfsdk.ModifyAttributePlanRequest, resp *tfsdk.ModifyAttributePlanResponse) {
	numberConfig := types.Bool{}
	diags := req.Config.GetAttribute(ctx, path.Root("number"), &numberConfig)
	resp.Diagnostics.Append(diags...)
	if resp.Diagnostics.HasError() {
		return
	}

	numericConfig := types.Bool{}
	diags = req.Config.GetAttribute(ctx, path.Root("numeric"), &numericConfig)
	resp.Diagnostics.Append(diags...)
	if resp.Diagnostics.HasError() {
		return
	}

	if !numberConfig.Null && !numericConfig.Null {
		resp.Diagnostics.AddError(
			"Number numeric attribute plan modifier failed",
			"Cannot specify both number and numeric in config",
		)
		return
	}

	// Default to true for both number and numeric when both are null.
	if numberConfig.Null && numericConfig.Null {
		resp.AttributePlan = types.Bool{Value: true}
		return
	}

	// Default to using value for numeric if number is null
	if numberConfig.Null && !numericConfig.Null {
		resp.AttributePlan = numericConfig
		return
	}

	// Default to using value for number if numeric is null
	if !numberConfig.Null && numericConfig.Null {
		resp.AttributePlan = numberConfig
		return
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Christian Gärtner. All rights reserved.
// Use of this source code is governed by a MIT license
// that can be found in the LICENSE file.

package build

import (
	"fmt"
	"errors"
	"github.com/achtern/kluver/lexer"
)

type Shader struct {
	version  string
	vertex   Tokens
	fragment Tokens
	global   Tokens
	compiled GLSL
}

type GLSL struct {
	vertex   string
	fragment string
	provides []Tokens
	requests []Tokens
}

type varDef struct {
	typ  string
	name string
}

type Tokens []lexer.Token

const providePlaceholder = "___PROVIDE___REPLACE___HERE___\n"

func (s *Shader) String() string {
	return fmt.Sprintf("Shader(%s, vertex=%q, fragment=%q, global=%q)", s.version, s.vertex, s.fragment, s.global)
}

func Build(tokenStream <-chan lexer.Token) (string, string, error) {
	global := make(Tokens, 0)
	vertex := make(Tokens, 0)
	fragment := make(Tokens, 0)

	shader := Shader{}

	// phase 0 : global
	// phase 1 : vertex
	// phase 2 : fragment
	phase := 0
	for token := range tokenStream {
		switch token.Typ {
		case lexer.TokenVertex:
			phase = 1
			continue
		case lexer.TokenFragment:
			phase = 2
			continue
		case lexer.TokenEnd:
			phase = 0
			continue
		case lexer.TokenVoid:
			continue
		}

		if token.Typ == lexer.TokenVersionNumber {
			shader.version = token.Val
			continue
		}

		if token.Typ == lexer.TokenAction {
			// we do need the action tokens after lexing
			continue
		}

		switch phase {
		case 0:
			global = append(global, token)
		case 1:
			vertex = append(vertex, token)
		case 2:
			fragment = append(fragment, token)
		default:
			panic("unknow phase")
		}
	}

	shader.global = global
	shader.vertex = vertex
	shader.fragment = fragment

	shader.buildVertex()
	shader.buildFragment()

	for _, request := range shader.compiled.requests {
		if !contains(shader.compiled.provides, request) {
			return "", "", errors.New("Missing @provide statement for <" + request[0].Val + " " + request[1].Val + ">")
		}
	}

	return shader.compiled.vertex, shader.compiled.fragment, nil
}

func (shader *Shader) buildVertex() {

	// vertex shader can only provide data to the fragment shader
	s, p, _ := buildGeneric(shader.vertex, shader.version)
	shader.compiled.vertex = s
	shader.compiled.provides = p
}

func (shader *Shader) buildFragment() {

	// fragment shader can only request data from the vertex shader
	s, _, r := buildGeneric(shader.fragment, shader.version)
	shader.compiled.fragment = s
	shader.compiled.requests = r
}
<commit_msg>Tokens buckets are now set on the shader struct directly<commit_after>// Copyright 2015 Christian Gärtner. All rights reserved.
// Use of this source code is governed by a MIT license
// that can be found in the LICENSE file.

package build

import (
	"fmt"
	"errors"
	"github.com/achtern/kluver/lexer"
)

type Shader struct {
	version  string
	vertex   Tokens
	fragment Tokens
	global   Tokens
	compiled GLSL
}

type GLSL struct {
	vertex   string
	fragment string
	provides []Tokens
	requests []Tokens
}

type varDef struct {
	typ  string
	name string
}

type Tokens []lexer.Token

const providePlaceholder = "___PROVIDE___REPLACE___HERE___\n"

func (s *Shader) String() string {
	return fmt.Sprintf("Shader(%s, vertex=%q, fragment=%q, global=%q)", s.version, s.vertex, s.fragment, s.global)
}

func Build(tokenStream <-chan lexer.Token) (string, string, error) {
	shader := Shader{}

	shader.global = make(Tokens, 0)
	shader.vertex = make(Tokens, 0)
	shader.fragment = make(Tokens, 0)

	// phase 0 : global
	// phase 1 : vertex
	// phase 2 : fragment
	phase := 0
	for token := range tokenStream {
		switch token.Typ {
		case lexer.TokenVertex:
			phase = 1
			continue
		case lexer.TokenFragment:
			phase = 2
			continue
		case lexer.TokenEnd:
			phase = 0
			continue
		case lexer.TokenVoid:
			continue
		}

		if token.Typ == lexer.TokenVersionNumber {
			shader.version = token.Val
			continue
		}

		if token.Typ == lexer.TokenAction {
			// we do need the action tokens after lexing
			continue
		}

		switch phase {
		case 0:
			shader.global = append(shader.global, token)
		case 1:
			shader.vertex = append(shader.vertex, token)
		case 2:
			shader.fragment = append(shader.fragment, token)
		default:
			panic("unknow phase")
		}
	}

	// shader.global = global
	// shader.vertex = vertex
	// shader.fragment = fragment

	shader.buildVertex()
	shader.buildFragment()

	for _, request := range shader.compiled.requests {
		if !contains(shader.compiled.provides, request) {
			return "", "", errors.New("Missing @provide statement for <" + request[0].Val + " " + request[1].Val + ">")
		}
	}

	return shader.compiled.vertex, shader.compiled.fragment, nil
}

func (shader *Shader) buildVertex() {

	// vertex shader can only provide data to the fragment shader
	s, p, _ := buildGeneric(shader.vertex, shader.version)
	shader.compiled.vertex = s
	shader.compiled.provides = p
}

func (shader *Shader) buildFragment() {

	// fragment shader can only request data from the vertex shader
	s, _, r := buildGeneric(shader.fragment, shader.version)
	shader.compiled.fragment = s
	shader.compiled.requests = r
}
<|endoftext|>


<commit_before>package core

import (
	"flag"
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"sync"
	"syscall"
	"time"

	"github.com/joyent/containerpilot/backends"
	"github.com/joyent/containerpilot/config"
	"github.com/joyent/containerpilot/discovery"
	"github.com/joyent/containerpilot/services"
	"github.com/joyent/containerpilot/tasks"
	"github.com/joyent/containerpilot/telemetry"
	"github.com/joyent/containerpilot/utils"

	log "github.com/Sirupsen/logrus"
)

var (
	// Version is the version for this build, set at build time via LDFLAGS
	Version string
	// GitHash is the short-form commit hash of this build, set at build time
	GitHash string
)

// App encapsulates the state of ContainerPilot after the initial setup.
// after it is run, it can be reloaded and paused with signals.
type App struct {
	DiscoveryService discovery.DiscoveryService
	Services         []*services.Service
	Backends         []*backends.Backend
	Tasks            []*tasks.Task
	Telemetry        *telemetry.Telemetry
	PreStartCmd      *exec.Cmd
	PreStopCmd       *exec.Cmd
	PostStopCmd      *exec.Cmd
	Command          *exec.Cmd
	StopTimeout      int
	QuitChannels     []chan bool
	maintModeLock    *sync.RWMutex
	signalLock       *sync.RWMutex
	paused           bool
	ConfigFlag       string
}

// EmptyApp creates an empty application
func EmptyApp() *App {
	app := &App{}
	app.maintModeLock = &sync.RWMutex{}
	app.signalLock = &sync.RWMutex{}
	return app
}

// LoadApp parses the commandline arguments and loads the config
func LoadApp() (*App, error) {

	var configFlag string
	var versionFlag bool

	if !flag.Parsed() {
		flag.StringVar(&configFlag, "config", "",
			"JSON config or file:// path to JSON config file.")
		flag.BoolVar(&versionFlag, "version", false, "Show version identifier and quit.")
		flag.Parse()
	}
	if versionFlag {
		fmt.Printf("Version: %s\nGitHash: %s\n", Version, GitHash)
		os.Exit(0)
	}
	if configFlag == "" {
		configFlag = os.Getenv("CONTAINERPILOT")
	}

	cfg, err := config.ParseConfig(configFlag)
	if err != nil {
		return nil, err
	}
	app, err := NewApp(cfg)
	if err != nil {
		return nil, err
	}
	return app, nil
}

// NewApp creates a new App from the config
func NewApp(cfg *config.Config) (*App, error) {
	a := EmptyApp()

	// onStart has been deprecated for preStart. Remove in 2.0
	if cfg.PreStart != nil && cfg.OnStart != nil {
		log.Warnf("The onStart option has been deprecated in favor of preStart. ContainerPilot will use only the preStart option provided")
	}

	// alias the onStart behavior to preStart
	if cfg.PreStart == nil && cfg.OnStart != nil {
		log.Warnf("The onStart option has been deprecated in favor of preStart. ContainerPilot will use the onStart option as a preStart")
		cfg.PreStart = cfg.OnStart
	}

	preStartCmd, err := cfg.ParsePreStart()
	if err != nil {
		return nil, err
	}
	a.PreStartCmd = preStartCmd

	preStopCmd, err := cfg.ParsePreStop()
	if err != nil {
		return nil, err
	}
	a.PreStopCmd = preStopCmd

	postStopCmd, err := cfg.ParsePostStop()
	if err != nil {
		return nil, err
	}
	a.PostStopCmd = postStopCmd

	stopTimeout, err := cfg.ParseStopTimeout()
	if err != nil {
		return nil, err
	}
	a.StopTimeout = stopTimeout

	discoveryService, err := cfg.ParseDiscoveryService()
	if err != nil {
		return nil, err
	}
	a.DiscoveryService = discoveryService

	backends, err := cfg.ParseBackends(discoveryService)
	if err != nil {
		return nil, err
	}
	a.Backends = backends

	services, err := cfg.ParseServices(discoveryService)
	if err != nil {
		return nil, err
	}
	a.Services = services

	telemetry, err := cfg.ParseTelemetry()
	if err != nil {
		return nil, err
	}

	if telemetry != nil {
		telemetryService, err2 := config.CreateTelemetryService(telemetry, discoveryService)
		if err2 != nil {
			return nil, err2
		}
		a.Telemetry = telemetry
		a.Services = append(a.Services, telemetryService)
	}

	tasks, err := cfg.ParseTasks()
	if err != nil {
		return nil, err
	}
	a.Tasks = tasks

	return a, nil
}

// Run starts the application and blocks until finished
func (a *App) Run() {
	// Run the preStart handler, if any, and exit if it returns an error
	if preStartCode, err := utils.RunWithFields(a.PreStartCmd, log.Fields{"process": "PreStart"}); err != nil {
		os.Exit(preStartCode)
	}

	// Set up handlers for polling and to accept signal interrupts
	if 1 == os.Getpid() {
		reapChildren()
	}
	a.handleSignals()
	a.handlePolling()

	if len(flag.Args()) != 0 {
		// Run our main application and capture its stdout/stderr.
		// This will block until the main application exits and then os.Exit
		// with the exit code of that application.
		a.Command = utils.ArgsToCmd(flag.Args())
		a.Command.Stderr = os.Stderr
		a.Command.Stdout = os.Stdout
		code, err := utils.ExecuteAndWait(a.Command)
		if err != nil {
			log.Println(err)
		}
		// Run the PostStop handler, if any, and exit if it returns an error
		if postStopCode, err := utils.RunWithFields(a.PostStopCmd, log.Fields{"process": "PostStop"}); err != nil {
			os.Exit(postStopCode)
		}
		os.Exit(code)
	}

	// block forever, as we're polling in the two polling functions and
	// did not os.Exit by waiting on an external application.
	select {}
}

// ToggleMaintenanceMode marks all services for maintenance
func (a *App) ToggleMaintenanceMode() {
	a.maintModeLock.RLock()
	a.signalLock.Lock()
	defer a.signalLock.Unlock()
	defer a.maintModeLock.RUnlock()
	a.paused = !a.paused
	if a.paused {
		a.forAllServices(markServiceForMaintenance)
	}
}

// InMaintenanceMode checks if the App is in maintenance mode
func (a *App) InMaintenanceMode() bool {
	// we wrap access to `paused` in a RLock so that if we're in the middle of
	// marking services for maintenance we don't get stale reads
	a.maintModeLock.RLock()
	defer a.maintModeLock.RUnlock()
	return a.paused
}

// Terminate kills the application
func (a *App) Terminate() {
	a.signalLock.Lock()
	defer a.signalLock.Unlock()
	a.stopPolling()
	a.forAllServices(deregisterService)

	// Run and wait for preStop command to exit
	utils.RunWithFields(a.PreStopCmd, log.Fields{"process": "PreStop"})

	cmd := a.Command
	if cmd == nil || cmd.Process == nil {
		// Not managing the process, so don't do anything
		return
	}
	if a.StopTimeout > 0 {
		if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
			log.Warnf("Error sending SIGTERM to application: %s", err)
		} else {
			time.AfterFunc(time.Duration(a.StopTimeout)*time.Second, func() {
				log.Infof("Killing Process %#v", cmd.Process)
				cmd.Process.Kill()
			})
			return
		}
	}
	log.Infof("Killing Process %#v", a.Command.Process)
	cmd.Process.Kill()
}

func (a *App) stopPolling() {
	for _, quit := range a.QuitChannels {
		quit <- true
	}
}

func markServiceForMaintenance(service *services.Service) {
	log.Infof("Marking for maintenance: %s", service.Name)
	service.MarkForMaintenance()
}

func deregisterService(service *services.Service) {
	log.Infof("Deregistering service: %s", service.Name)
	service.Deregister()
}

// Reload will try to update the running application by
// loading the config and applying changes to the services
// A reload cannot change the shimmed application, or the preStart script
func (a *App) Reload() error {
	a.signalLock.Lock()
	defer a.signalLock.Unlock()
	log.Infof("Reloading configuration.")

	newCfg, err := config.ParseConfig(a.ConfigFlag)
	if err != nil {
		log.Errorf("Error parsing config: %v", err)
		return err
	}
	newApp, err := NewApp(newCfg)
	if err != nil {
		log.Errorf("Error initializing config: %v", err)
		return err
	}

	a.stopPolling()
	a.forAllServices(deregisterService)
	signal.Reset()

	a.load(newApp)
	return nil
}

func (a *App) load(newApp *App) {
	a.DiscoveryService = newApp.DiscoveryService
	a.PostStopCmd = newApp.PostStopCmd
	a.PreStopCmd = newApp.PreStopCmd
	a.Services = newApp.Services
	a.Backends = newApp.Backends
	a.StopTimeout = newApp.StopTimeout
	if a.Telemetry != nil {
		a.Telemetry.Shutdown()
	}
	a.Telemetry = newApp.Telemetry
	a.Tasks = newApp.Tasks
	a.handleSignals()
	a.handlePolling()
}

type serviceFunc func(service *services.Service)

func (a *App) forAllServices(fn serviceFunc) {
	for _, service := range a.Services {
		fn(service)
	}
}

// HandlePolling sets up polling functions and write their quit channels
// back to our config
func (a *App) handlePolling() {
	var quit []chan bool
	for _, backend := range a.Backends {
		quit = append(quit, a.poll(backend))
	}
	for _, service := range a.Services {
		quit = append(quit, a.poll(service))
	}
	if a.Telemetry != nil {
		for _, sensor := range a.Telemetry.Sensors {
			quit = append(quit, a.poll(sensor))
		}
		a.Telemetry.Serve()
	}
	if a.Tasks != nil {
		for _, task := range a.Tasks {
			quit = append(quit, a.poll(task))
		}
	}
	a.QuitChannels = quit
}
<commit_msg>fix: run preStart after handleSignals<commit_after>package core

import (
	"flag"
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"sync"
	"syscall"
	"time"

	"github.com/joyent/containerpilot/backends"
	"github.com/joyent/containerpilot/config"
	"github.com/joyent/containerpilot/discovery"
	"github.com/joyent/containerpilot/services"
	"github.com/joyent/containerpilot/tasks"
	"github.com/joyent/containerpilot/telemetry"
	"github.com/joyent/containerpilot/utils"

	log "github.com/Sirupsen/logrus"
)

var (
	// Version is the version for this build, set at build time via LDFLAGS
	Version string
	// GitHash is the short-form commit hash of this build, set at build time
	GitHash string
)

// App encapsulates the state of ContainerPilot after the initial setup.
// after it is run, it can be reloaded and paused with signals.
type App struct {
	DiscoveryService discovery.DiscoveryService
	Services         []*services.Service
	Backends         []*backends.Backend
	Tasks            []*tasks.Task
	Telemetry        *telemetry.Telemetry
	PreStartCmd      *exec.Cmd
	PreStopCmd       *exec.Cmd
	PostStopCmd      *exec.Cmd
	Command          *exec.Cmd
	StopTimeout      int
	QuitChannels     []chan bool
	maintModeLock    *sync.RWMutex
	signalLock       *sync.RWMutex
	paused           bool
	ConfigFlag       string
}

// EmptyApp creates an empty application
func EmptyApp() *App {
	app := &App{}
	app.maintModeLock = &sync.RWMutex{}
	app.signalLock = &sync.RWMutex{}
	return app
}

// LoadApp parses the commandline arguments and loads the config
func LoadApp() (*App, error) {

	var configFlag string
	var versionFlag bool

	if !flag.Parsed() {
		flag.StringVar(&configFlag, "config", "",
			"JSON config or file:// path to JSON config file.")
		flag.BoolVar(&versionFlag, "version", false, "Show version identifier and quit.")
		flag.Parse()
	}
	if versionFlag {
		fmt.Printf("Version: %s\nGitHash: %s\n", Version, GitHash)
		os.Exit(0)
	}
	if configFlag == "" {
		configFlag = os.Getenv("CONTAINERPILOT")
	}

	cfg, err := config.ParseConfig(configFlag)
	if err != nil {
		return nil, err
	}
	app, err := NewApp(cfg)
	if err != nil {
		return nil, err
	}
	return app, nil
}

// NewApp creates a new App from the config
func NewApp(cfg *config.Config) (*App, error) {
	a := EmptyApp()

	// onStart has been deprecated for preStart. Remove in 2.0
	if cfg.PreStart != nil && cfg.OnStart != nil {
		log.Warnf("The onStart option has been deprecated in favor of preStart. ContainerPilot will use only the preStart option provided")
	}

	// alias the onStart behavior to preStart
	if cfg.PreStart == nil && cfg.OnStart != nil {
		log.Warnf("The onStart option has been deprecated in favor of preStart. ContainerPilot will use the onStart option as a preStart")
		cfg.PreStart = cfg.OnStart
	}

	preStartCmd, err := cfg.ParsePreStart()
	if err != nil {
		return nil, err
	}
	a.PreStartCmd = preStartCmd

	preStopCmd, err := cfg.ParsePreStop()
	if err != nil {
		return nil, err
	}
	a.PreStopCmd = preStopCmd

	postStopCmd, err := cfg.ParsePostStop()
	if err != nil {
		return nil, err
	}
	a.PostStopCmd = postStopCmd

	stopTimeout, err := cfg.ParseStopTimeout()
	if err != nil {
		return nil, err
	}
	a.StopTimeout = stopTimeout

	discoveryService, err := cfg.ParseDiscoveryService()
	if err != nil {
		return nil, err
	}
	a.DiscoveryService = discoveryService

	backends, err := cfg.ParseBackends(discoveryService)
	if err != nil {
		return nil, err
	}
	a.Backends = backends

	services, err := cfg.ParseServices(discoveryService)
	if err != nil {
		return nil, err
	}
	a.Services = services

	telemetry, err := cfg.ParseTelemetry()
	if err != nil {
		return nil, err
	}

	if telemetry != nil {
		telemetryService, err2 := config.CreateTelemetryService(telemetry, discoveryService)
		if err2 != nil {
			return nil, err2
		}
		a.Telemetry = telemetry
		a.Services = append(a.Services, telemetryService)
	}

	tasks, err := cfg.ParseTasks()
	if err != nil {
		return nil, err
	}
	a.Tasks = tasks

	return a, nil
}

// Run starts the application and blocks until finished
func (a *App) Run() {
	// Set up handlers for polling and to accept signal interrupts
	if 1 == os.Getpid() {
		reapChildren()
	}
	a.handleSignals()
	// Run the preStart handler, if any, and exit if it returns an error
	if preStartCode, err := utils.RunWithFields(a.PreStartCmd, log.Fields{"process": "PreStart"}); err != nil {
		os.Exit(preStartCode)
	}
	a.handlePolling()

	if len(flag.Args()) != 0 {
		// Run our main application and capture its stdout/stderr.
		// This will block until the main application exits and then os.Exit
		// with the exit code of that application.
		a.Command = utils.ArgsToCmd(flag.Args())
		a.Command.Stderr = os.Stderr
		a.Command.Stdout = os.Stdout
		code, err := utils.ExecuteAndWait(a.Command)
		if err != nil {
			log.Println(err)
		}
		// Run the PostStop handler, if any, and exit if it returns an error
		if postStopCode, err := utils.RunWithFields(a.PostStopCmd, log.Fields{"process": "PostStop"}); err != nil {
			os.Exit(postStopCode)
		}
		os.Exit(code)
	}

	// block forever, as we're polling in the two polling functions and
	// did not os.Exit by waiting on an external application.
	select {}
}

// ToggleMaintenanceMode marks all services for maintenance
func (a *App) ToggleMaintenanceMode() {
	a.maintModeLock.RLock()
	a.signalLock.Lock()
	defer a.signalLock.Unlock()
	defer a.maintModeLock.RUnlock()
	a.paused = !a.paused
	if a.paused {
		a.forAllServices(markServiceForMaintenance)
	}
}

// InMaintenanceMode checks if the App is in maintenance mode
func (a *App) InMaintenanceMode() bool {
	// we wrap access to `paused` in a RLock so that if we're in the middle of
	// marking services for maintenance we don't get stale reads
	a.maintModeLock.RLock()
	defer a.maintModeLock.RUnlock()
	return a.paused
}

// Terminate kills the application
func (a *App) Terminate() {
	a.signalLock.Lock()
	defer a.signalLock.Unlock()
	a.stopPolling()
	a.forAllServices(deregisterService)

	// Run and wait for preStop command to exit
	utils.RunWithFields(a.PreStopCmd, log.Fields{"process": "PreStop"})

	cmd := a.Command
	if cmd == nil || cmd.Process == nil {
		// Not managing the process, so don't do anything
		return
	}
	if a.StopTimeout > 0 {
		if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
			log.Warnf("Error sending SIGTERM to application: %s", err)
		} else {
			time.AfterFunc(time.Duration(a.StopTimeout)*time.Second, func() {
				log.Infof("Killing Process %#v", cmd.Process)
				cmd.Process.Kill()
			})
			return
		}
	}
	log.Infof("Killing Process %#v", a.Command.Process)
	cmd.Process.Kill()
}

func (a *App) stopPolling() {
	for _, quit := range a.QuitChannels {
		quit <- true
	}
}

func markServiceForMaintenance(service *services.Service) {
	log.Infof("Marking for maintenance: %s", service.Name)
	service.MarkForMaintenance()
}

func deregisterService(service *services.Service) {
	log.Infof("Deregistering service: %s", service.Name)
	service.Deregister()
}

// Reload will try to update the running application by
// loading the config and applying changes to the services
// A reload cannot change the shimmed application, or the preStart script
func (a *App) Reload() error {
	a.signalLock.Lock()
	defer a.signalLock.Unlock()
	log.Infof("Reloading configuration.")

	newCfg, err := config.ParseConfig(a.ConfigFlag)
	if err != nil {
		log.Errorf("Error parsing config: %v", err)
		return err
	}
	newApp, err := NewApp(newCfg)
	if err != nil {
		log.Errorf("Error initializing config: %v", err)
		return err
	}

	a.stopPolling()
	a.forAllServices(deregisterService)
	signal.Reset()

	a.load(newApp)
	return nil
}

func (a *App) load(newApp *App) {
	a.DiscoveryService = newApp.DiscoveryService
	a.PostStopCmd = newApp.PostStopCmd
	a.PreStopCmd = newApp.PreStopCmd
	a.Services = newApp.Services
	a.Backends = newApp.Backends
	a.StopTimeout = newApp.StopTimeout
	if a.Telemetry != nil {
		a.Telemetry.Shutdown()
	}
	a.Telemetry = newApp.Telemetry
	a.Tasks = newApp.Tasks
	a.handleSignals()
	a.handlePolling()
}

type serviceFunc func(service *services.Service)

func (a *App) forAllServices(fn serviceFunc) {
	for _, service := range a.Services {
		fn(service)
	}
}

// HandlePolling sets up polling functions and write their quit channels
// back to our config
func (a *App) handlePolling() {
	var quit []chan bool
	for _, backend := range a.Backends {
		quit = append(quit, a.poll(backend))
	}
	for _, service := range a.Services {
		quit = append(quit, a.poll(service))
	}
	if a.Telemetry != nil {
		for _, sensor := range a.Telemetry.Sensors {
			quit = append(quit, a.poll(sensor))
		}
		a.Telemetry.Serve()
	}
	if a.Tasks != nil {
		for _, task := range a.Tasks {
			quit = append(quit, a.poll(task))
		}
	}
	a.QuitChannels = quit
}
<|endoftext|>


<commit_before>package multiline

import (
	"regexp"
	"sync"
	"testing"
	"time"

	"github.com/fsouza/go-dockerclient"
	"github.com/gliderlabs/logspout/router"
	"strings"
)

type dummyAdapter struct {
	messages []*router.Message
	*sync.WaitGroup
}

type testData struct {
	input          []string
	expected       []string
	pattern        *regexp.Regexp
	matchFirstLine bool
	negateMatch    bool
}

func (da *dummyAdapter) Stream(logstream chan *router.Message) {
	for m := range logstream {
		da.messages = append(da.messages, m)
	}
	da.Done()
}

var tests = []*testData{
	{
		input: []string{
			"some",
			"  multi",
			"  line",
			"other",
			"  multiline",
		},
		expected: []string{
			"some\n  multi\n  line",
			"other\n  multiline",
		},
		pattern:        regexp.MustCompile(`^\s`),
		matchFirstLine: true,
		negateMatch:    true,
	},
	{
		input: []string{
			"some:",
			"multi",
			"line",
			"other:",
			"multiline",
		},
		expected: []string{
			"some:\nmulti\nline",
			"other:\nmultiline",
		},
		pattern:        regexp.MustCompile(`:$`),
		matchFirstLine: true,
		negateMatch:    false,
	},
	{
		input: []string{
			"some$",
			"multi$",
			"line",
			"other$",
			"multiline",
		},
		expected: []string{
			"some$\nmulti$\nline",
			"other$\nmultiline",
		},
		pattern:        regexp.MustCompile(`\$$`),
		matchFirstLine: false,
		negateMatch:    true,
	},
	{
		input: []string{
			"some",
			"multi",
			"line!",
			"other",
			"multiline!",
		},
		expected: []string{
			"some\nmulti\nline!",
			"other\nmultiline!",
		},
		pattern:        regexp.MustCompile(`!$`),
		matchFirstLine: false,
		negateMatch:    false,
	},
}

func TestMultiline(t *testing.T) {
	for _, test := range tests {
		in := make(chan *router.Message)
		out := make(chan *router.Message)
		container := &docker.Container{
			ID:     "test",
			Config: &docker.Config{},
		}

		da := &dummyAdapter{make([]*router.Message, 0), &sync.WaitGroup{}}
		da.Add(1)

		ma := &Adapter{
			out:             out,
			subAdapter:      da,
			enableByDefault: true,
			pattern:         test.pattern,
			matchFirstLine:  test.matchFirstLine,
			negateMatch:     test.negateMatch,
			flushAfter:      time.Millisecond * 200,
			checkInterval:   time.Microsecond * 100,
			buffers:         make(map[string]*router.Message),
			nextCheck:       time.After(time.Microsecond * 100),
		}

		go ma.Stream(in)

		for _, i := range test.input {
			in <- &router.Message{
				Container: container,
				Data:      i,
				Source:    "stdout",
				Time:      time.Now(),
			}
		}

		close(in)
		da.Wait()

		for i, m := range da.messages {
			if m.Data != test.expected[i] {
				t.Errorf("Expected: '%v', Got: '%v'", replaceNewLines(test.expected[i]), replaceNewLines(m.Data))
			}
		}
	}
}

func replaceNewLines(str string) string {
	return strings.Replace(str, "\n", "\\n", -1)
}<commit_msg>Fixed multiline adapter test<commit_after>package multiline

import (
	"regexp"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/fsouza/go-dockerclient"
	"github.com/gliderlabs/logspout/router"
)

type dummyAdapter struct {
	messages []*router.Message
	*sync.WaitGroup
}

type testData struct {
	input          []string
	expected       []string
	pattern        *regexp.Regexp
	matchFirstLine bool
	negateMatch    bool
}

func (da *dummyAdapter) Stream(logstream chan *router.Message) {
	for m := range logstream {
		da.messages = append(da.messages, m)
	}
	da.Done()
}

var tests = []*testData{
	{
		input: []string{
			"some",
			"  multi",
			"  line",
			"other",
			"  multiline",
		},
		expected: []string{
			"some\n  multi\n  line",
			"other\n  multiline",
		},
		pattern:        regexp.MustCompile(`^\s`),
		matchFirstLine: true,
		negateMatch:    true,
	},
	{
		input: []string{
			"some:",
			"multi",
			"line",
			"other:",
			"multiline",
		},
		expected: []string{
			"some:\nmulti\nline",
			"other:\nmultiline",
		},
		pattern:        regexp.MustCompile(`:$`),
		matchFirstLine: true,
		negateMatch:    false,
	},
	{
		input: []string{
			"some$",
			"multi$",
			"line",
			"other$",
			"multiline",
		},
		expected: []string{
			"some$\nmulti$\nline",
			"other$\nmultiline",
		},
		pattern:        regexp.MustCompile(`\$$`),
		matchFirstLine: false,
		negateMatch:    true,
	},
	{
		input: []string{
			"some",
			"multi",
			"line!",
			"other",
			"multiline!",
		},
		expected: []string{
			"some\nmulti\nline!",
			"other\nmultiline!",
		},
		pattern:        regexp.MustCompile(`!$`),
		matchFirstLine: false,
		negateMatch:    false,
	},
}

func TestMultiline(t *testing.T) {
	for _, test := range tests {
		in := make(chan *router.Message)
		out := make(chan *router.Message)
		container := &docker.Container{
			ID:     "test",
			Config: &docker.Config{},
		}

		da := &dummyAdapter{make([]*router.Message, 0), &sync.WaitGroup{}}
		da.Add(1)

		ma := &Adapter{
			out:             out,
			subAdapter:      da,
			enableByDefault: true,
			pattern:         test.pattern,
			matchFirstLine:  test.matchFirstLine,
			negateMatch:     test.negateMatch,
			flushAfter:      time.Millisecond * 200,
			checkInterval:   time.Microsecond * 100,
			buffers:         make(map[string]*router.Message),
			nextCheck:       time.After(time.Microsecond * 100),
			separator:       "\n",
		}

		go ma.Stream(in)

		for _, i := range test.input {
			in <- &router.Message{
				Container: container,
				Data:      i,
				Source:    "stdout",
				Time:      time.Now(),
			}
		}

		close(in)
		da.Wait()

		for i, m := range da.messages {
			if m.Data != test.expected[i] {
				t.Errorf("Expected: '%v', Got: '%v'", replaceNewLines(test.expected[i]), replaceNewLines(m.Data))
			}
		}
	}
}

func replaceNewLines(str string) string {
	return strings.Replace(str, "\n", "\\n", -1)
}
<|endoftext|>


<commit_before>// Copyright 2017 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://wwm.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package ddl

import (
	"context"
	"sync/atomic"
	"time"

	"github.com/pingcap/errors"
	"github.com/pingcap/parser/ast"
	"github.com/pingcap/parser/charset"
	"github.com/pingcap/parser/model"
	"github.com/pingcap/tidb/ddl/util"
	"github.com/pingcap/tidb/sessionctx"
	"go.etcd.io/etcd/clientv3"
)

var _ util.SchemaSyncer = &MockSchemaSyncer{}

const mockCheckVersInterval = 2 * time.Millisecond

// MockSchemaSyncer is a mock schema syncer, it is exported for tesing.
type MockSchemaSyncer struct {
	selfSchemaVersion int64
	globalVerCh       chan clientv3.WatchResponse
	mockSession       chan struct{}
}

// NewMockSchemaSyncer creates a new mock SchemaSyncer.
func NewMockSchemaSyncer() util.SchemaSyncer {
	return &MockSchemaSyncer{}
}

// Init implements SchemaSyncer.Init interface.
func (s *MockSchemaSyncer) Init(ctx context.Context) error {
	s.globalVerCh = make(chan clientv3.WatchResponse, 1)
	s.mockSession = make(chan struct{}, 1)
	return nil
}

// GlobalVersionCh implements SchemaSyncer.GlobalVersionCh interface.
func (s *MockSchemaSyncer) GlobalVersionCh() clientv3.WatchChan {
	return s.globalVerCh
}

// WatchGlobalSchemaVer implements SchemaSyncer.WatchGlobalSchemaVer interface.
func (s *MockSchemaSyncer) WatchGlobalSchemaVer(context.Context) {}

// UpdateSelfVersion implements SchemaSyncer.UpdateSelfVersion interface.
func (s *MockSchemaSyncer) UpdateSelfVersion(ctx context.Context, version int64) error {
	atomic.StoreInt64(&s.selfSchemaVersion, version)
	return nil
}

// Done implements SchemaSyncer.Done interface.
func (s *MockSchemaSyncer) Done() <-chan struct{} {
	return s.mockSession
}

// CloseSession mockSession, it is exported for testing.
func (s *MockSchemaSyncer) CloseSession() {
	close(s.mockSession)
}

// Restart implements SchemaSyncer.Restart interface.
func (s *MockSchemaSyncer) Restart(_ context.Context) error {
	s.mockSession = make(chan struct{}, 1)
	return nil
}

// OwnerUpdateGlobalVersion implements SchemaSyncer.OwnerUpdateGlobalVersion interface.
func (s *MockSchemaSyncer) OwnerUpdateGlobalVersion(ctx context.Context, version int64) error {
	select {
	case s.globalVerCh <- clientv3.WatchResponse{}:
	default:
	}
	return nil
}

// MustGetGlobalVersion implements SchemaSyncer.MustGetGlobalVersion interface.
func (s *MockSchemaSyncer) MustGetGlobalVersion(ctx context.Context) (int64, error) {
	return 0, nil
}

// OwnerCheckAllVersions implements SchemaSyncer.OwnerCheckAllVersions interface.
func (s *MockSchemaSyncer) OwnerCheckAllVersions(ctx context.Context, latestVer int64) error {
	ticker := time.NewTicker(mockCheckVersInterval)
	defer ticker.Stop()

	for {
		select {
		case <-ctx.Done():
			return errors.Trace(ctx.Err())
		case <-ticker.C:
			ver := atomic.LoadInt64(&s.selfSchemaVersion)
			if ver == latestVer {
				return nil
			}
		}
	}
}

// NotifyCleanExpiredPaths implements SchemaSyncer.NotifyCleanExpiredPaths interface.
func (s *MockSchemaSyncer) NotifyCleanExpiredPaths() bool { return true }

// StartCleanWork implements SchemaSyncer.StartCleanWork interface.
func (s *MockSchemaSyncer) StartCleanWork() {}

// Close implements SchemaSyncer.Close interface.
func (s *MockSchemaSyncer) Close() {}

type mockDelRange struct {
}

// newMockDelRangeManager creates a mock delRangeManager only used for test.
func newMockDelRangeManager() delRangeManager {
	return &mockDelRange{}
}

// addDelRangeJob implements delRangeManager interface.
func (dr *mockDelRange) addDelRangeJob(job *model.Job) error {
	return nil
}

// removeFromGCDeleteRange implements delRangeManager interface.
func (dr *mockDelRange) removeFromGCDeleteRange(jobID int64, tableIDs []int64) error {
	return nil
}

// start implements delRangeManager interface.
func (dr *mockDelRange) start() {}

// clear implements delRangeManager interface.
func (dr *mockDelRange) clear() {}

// MockTableInfo mocks a table info by create table stmt ast and a specified table id.
func MockTableInfo(ctx sessionctx.Context, stmt *ast.CreateTableStmt, tableID int64) (*model.TableInfo, error) {
	chs, coll := charset.GetDefaultCharsetAndCollate()
	cols, newConstraints, err := buildColumnsAndConstraints(ctx, stmt.Cols, stmt.Constraints, chs, coll)
	if err != nil {
		return nil, errors.Trace(err)
	}
	tbl, err := buildTableInfo(ctx, stmt.Table.Name, cols, newConstraints, "", "")
	if err != nil {
		return nil, errors.Trace(err)
	}
	tbl.ID = tableID

	// The specified charset will be handled in handleTableOptions
	if err = handleTableOptions(stmt.Options, tbl); err != nil {
		return nil, errors.Trace(err)
	}

	return tbl, nil
}
<commit_msg>ddl: fix mix DDLs blocks 2*lease on unistore (#18468)<commit_after>// Copyright 2017 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://wwm.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package ddl

import (
	"context"
	"sync/atomic"
	"time"

	"github.com/pingcap/errors"
	"github.com/pingcap/parser/ast"
	"github.com/pingcap/parser/charset"
	"github.com/pingcap/parser/model"
	"github.com/pingcap/tidb/ddl/util"
	"github.com/pingcap/tidb/sessionctx"
	"go.etcd.io/etcd/clientv3"
)

var _ util.SchemaSyncer = &MockSchemaSyncer{}

const mockCheckVersInterval = 2 * time.Millisecond

// MockSchemaSyncer is a mock schema syncer, it is exported for tesing.
type MockSchemaSyncer struct {
	selfSchemaVersion int64
	globalVerCh       chan clientv3.WatchResponse
	mockSession       chan struct{}
}

// NewMockSchemaSyncer creates a new mock SchemaSyncer.
func NewMockSchemaSyncer() util.SchemaSyncer {
	return &MockSchemaSyncer{}
}

// Init implements SchemaSyncer.Init interface.
func (s *MockSchemaSyncer) Init(ctx context.Context) error {
	s.globalVerCh = make(chan clientv3.WatchResponse, 1)
	s.mockSession = make(chan struct{}, 1)
	return nil
}

// GlobalVersionCh implements SchemaSyncer.GlobalVersionCh interface.
func (s *MockSchemaSyncer) GlobalVersionCh() clientv3.WatchChan {
	return s.globalVerCh
}

// WatchGlobalSchemaVer implements SchemaSyncer.WatchGlobalSchemaVer interface.
func (s *MockSchemaSyncer) WatchGlobalSchemaVer(context.Context) {}

// UpdateSelfVersion implements SchemaSyncer.UpdateSelfVersion interface.
func (s *MockSchemaSyncer) UpdateSelfVersion(ctx context.Context, version int64) error {
	atomic.StoreInt64(&s.selfSchemaVersion, version)
	return nil
}

// Done implements SchemaSyncer.Done interface.
func (s *MockSchemaSyncer) Done() <-chan struct{} {
	return s.mockSession
}

// CloseSession mockSession, it is exported for testing.
func (s *MockSchemaSyncer) CloseSession() {
	close(s.mockSession)
}

// Restart implements SchemaSyncer.Restart interface.
func (s *MockSchemaSyncer) Restart(_ context.Context) error {
	s.mockSession = make(chan struct{}, 1)
	return nil
}

// OwnerUpdateGlobalVersion implements SchemaSyncer.OwnerUpdateGlobalVersion interface.
func (s *MockSchemaSyncer) OwnerUpdateGlobalVersion(ctx context.Context, version int64) error {
	select {
	case s.globalVerCh <- clientv3.WatchResponse{}:
	default:
	}
	return nil
}

// MustGetGlobalVersion implements SchemaSyncer.MustGetGlobalVersion interface.
func (s *MockSchemaSyncer) MustGetGlobalVersion(ctx context.Context) (int64, error) {
	return 0, nil
}

// OwnerCheckAllVersions implements SchemaSyncer.OwnerCheckAllVersions interface.
func (s *MockSchemaSyncer) OwnerCheckAllVersions(ctx context.Context, latestVer int64) error {
	ticker := time.NewTicker(mockCheckVersInterval)
	defer ticker.Stop()

	for {
		select {
		case <-ctx.Done():
			return errors.Trace(ctx.Err())
		case <-ticker.C:
			ver := atomic.LoadInt64(&s.selfSchemaVersion)
			if ver >= latestVer {
				return nil
			}
		}
	}
}

// NotifyCleanExpiredPaths implements SchemaSyncer.NotifyCleanExpiredPaths interface.
func (s *MockSchemaSyncer) NotifyCleanExpiredPaths() bool { return true }

// StartCleanWork implements SchemaSyncer.StartCleanWork interface.
func (s *MockSchemaSyncer) StartCleanWork() {}

// Close implements SchemaSyncer.Close interface.
func (s *MockSchemaSyncer) Close() {}

type mockDelRange struct {
}

// newMockDelRangeManager creates a mock delRangeManager only used for test.
func newMockDelRangeManager() delRangeManager {
	return &mockDelRange{}
}

// addDelRangeJob implements delRangeManager interface.
func (dr *mockDelRange) addDelRangeJob(job *model.Job) error {
	return nil
}

// removeFromGCDeleteRange implements delRangeManager interface.
func (dr *mockDelRange) removeFromGCDeleteRange(jobID int64, tableIDs []int64) error {
	return nil
}

// start implements delRangeManager interface.
func (dr *mockDelRange) start() {}

// clear implements delRangeManager interface.
func (dr *mockDelRange) clear() {}

// MockTableInfo mocks a table info by create table stmt ast and a specified table id.
func MockTableInfo(ctx sessionctx.Context, stmt *ast.CreateTableStmt, tableID int64) (*model.TableInfo, error) {
	chs, coll := charset.GetDefaultCharsetAndCollate()
	cols, newConstraints, err := buildColumnsAndConstraints(ctx, stmt.Cols, stmt.Constraints, chs, coll)
	if err != nil {
		return nil, errors.Trace(err)
	}
	tbl, err := buildTableInfo(ctx, stmt.Table.Name, cols, newConstraints, "", "")
	if err != nil {
		return nil, errors.Trace(err)
	}
	tbl.ID = tableID

	// The specified charset will be handled in handleTableOptions
	if err = handleTableOptions(stmt.Options, tbl); err != nil {
		return nil, errors.Trace(err)
	}

	return tbl, nil
}
<|endoftext|>


<commit_before>/*
Copyright IBM Corp. 2017 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

                 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package blocksprovider

import (
	"math"
	"time"

	"sync/atomic"

	"github.com/golang/protobuf/proto"
	gossipcommon "github.com/hyperledger/fabric/gossip/common"
	"github.com/hyperledger/fabric/gossip/discovery"

	"github.com/hyperledger/fabric/common/flogging"
	"github.com/hyperledger/fabric/gossip/api"
	"github.com/hyperledger/fabric/protos/common"
	gossip_proto "github.com/hyperledger/fabric/protos/gossip"
	"github.com/hyperledger/fabric/protos/orderer"
	"github.com/op/go-logging"
)

// LedgerInfo an adapter to provide the interface to query
// the ledger committer for current ledger height
type LedgerInfo interface {
	// LedgerHeight returns current local ledger height
	LedgerHeight() (uint64, error)
}

// GossipServiceAdapter serves to provide basic functionality
// required from gossip service by delivery service
type GossipServiceAdapter interface {
	// PeersOfChannel returns slice with members of specified channel
	PeersOfChannel(gossipcommon.ChainID) []discovery.NetworkMember

	// AddPayload adds payload to the local state sync buffer
	AddPayload(chainID string, payload *gossip_proto.Payload) error

	// Gossip the message across the peers
	Gossip(msg *gossip_proto.GossipMessage)
}

// BlocksProvider used to read blocks from the ordering service
// for specified chain it subscribed to
type BlocksProvider interface {
	// DeliverBlocks starts delivering and disseminating blocks
	DeliverBlocks()

	// Stop shutdowns blocks provider and stops delivering new blocks
	Stop()
}

// BlocksDeliverer defines interface which actually helps
// to abstract the AtomicBroadcast_DeliverClient with only
// required method for blocks provider.
// This also decouples the production implementation of the gRPC stream
// from the code in order for the code to be more modular and testable.
type BlocksDeliverer interface {
	// Recv retrieves a response from the ordering service
	Recv() (*orderer.DeliverResponse, error)

	// Send sends an envelope to the ordering service
	Send(*common.Envelope) error
}

type streamClient interface {
	BlocksDeliverer

	// Close closes the stream and its underlying connection
	Close()

	// Disconnect disconnects from the remote node
	Disconnect()
}

// blocksProviderImpl the actual implementation for BlocksProvider interface
type blocksProviderImpl struct {
	chainID string

	client streamClient

	gossip GossipServiceAdapter

	mcs api.MessageCryptoService

	done int32

	wrongStatusThreshold int
}

const wrongStatusThreshold = 10

var maxRetryDelay = time.Second * 10

var logger *logging.Logger // package-level logger

func init() {
	logger = flogging.MustGetLogger("blocksProvider")
}

// NewBlocksProvider constructor function to create blocks deliverer instance
func NewBlocksProvider(chainID string, client streamClient, gossip GossipServiceAdapter, mcs api.MessageCryptoService) BlocksProvider {
	return &blocksProviderImpl{
		chainID:              chainID,
		client:               client,
		gossip:               gossip,
		mcs:                  mcs,
		wrongStatusThreshold: wrongStatusThreshold,
	}
}

// DeliverBlocks used to pull out blocks from the ordering service to
// distributed them across peers
func (b *blocksProviderImpl) DeliverBlocks() {
	errorStatusCounter := 0
	statusCounter := 0
	defer b.client.Close()
	for !b.isDone() {
		msg, err := b.client.Recv()
		if err != nil {
			logger.Warningf("[%s] Receive error: %s", b.chainID, err.Error())
			return
		}
		switch t := msg.Type.(type) {
		case *orderer.DeliverResponse_Status:
			if t.Status == common.Status_SUCCESS {
				logger.Warningf("[%s] ERROR! Received success for a seek that should never complete", b.chainID)
				return
			}
			if t.Status == common.Status_BAD_REQUEST || t.Status == common.Status_FORBIDDEN {
				logger.Errorf("[%s] Got error %v", b.chainID, t)
				errorStatusCounter++
				if errorStatusCounter > b.wrongStatusThreshold {
					logger.Criticalf("[%s] Wrong statuses threshold passed, stopping block provider", b.chainID)
					return
				}
			} else {
				errorStatusCounter = 0
				logger.Warningf("[%s] Got error %v", b.chainID, t)
			}
			maxDelay := float64(maxRetryDelay)
			currDelay := float64(time.Duration(math.Pow(2, float64(statusCounter))) * 100 * time.Millisecond)
			time.Sleep(time.Duration(math.Min(maxDelay, currDelay)))
			statusCounter++
			b.client.Disconnect()
			continue
		case *orderer.DeliverResponse_Block:
			errorStatusCounter = 0
			statusCounter = 0
			seqNum := t.Block.Header.Number

			marshaledBlock, err := proto.Marshal(t.Block)
			if err != nil {
				logger.Errorf("[%s] Error serializing block with sequence number %d, due to %s", b.chainID, seqNum, err)
				continue
			}
			if err := b.mcs.VerifyBlock(gossipcommon.ChainID(b.chainID), seqNum, marshaledBlock); err != nil {
				logger.Errorf("[%s] Error verifying block with sequnce number %d, due to %s", b.chainID, seqNum, err)
				continue
			}

			numberOfPeers := len(b.gossip.PeersOfChannel(gossipcommon.ChainID(b.chainID)))
			// Create payload with a block received
			payload := createPayload(seqNum, marshaledBlock)
			// Use payload to create gossip message
			gossipMsg := createGossipMsg(b.chainID, payload)

			logger.Debugf("[%s] Adding payload locally, buffer seqNum = [%d], peers number [%d]", b.chainID, seqNum, numberOfPeers)
			// Add payload to local state payloads buffer
			b.gossip.AddPayload(b.chainID, payload)

			// Gossip messages with other nodes
			logger.Debugf("[%s] Gossiping block [%d], peers number [%d]", b.chainID, seqNum, numberOfPeers)
			b.gossip.Gossip(gossipMsg)
		default:
			logger.Warningf("[%s] Received unknown: ", b.chainID, t)
			return
		}
	}
}

// Stop stops blocks delivery provider
func (b *blocksProviderImpl) Stop() {
	atomic.StoreInt32(&b.done, 1)
	b.client.Close()
}

// Check whenever provider is stopped
func (b *blocksProviderImpl) isDone() bool {
	return atomic.LoadInt32(&b.done) == 1
}

func createGossipMsg(chainID string, payload *gossip_proto.Payload) *gossip_proto.GossipMessage {
	gossipMsg := &gossip_proto.GossipMessage{
		Nonce:   0,
		Tag:     gossip_proto.GossipMessage_CHAN_AND_ORG,
		Channel: []byte(chainID),
		Content: &gossip_proto.GossipMessage_DataMsg{
			DataMsg: &gossip_proto.DataMessage{
				Payload: payload,
			},
		},
	}
	return gossipMsg
}

func createPayload(seqNum uint64, marshaledBlock []byte) *gossip_proto.Payload {
	return &gossip_proto.Payload{
		Data:   marshaledBlock,
		SeqNum: seqNum,
	}
}
<commit_msg>[FAB-4613] Warning logs flooding in orderer and peers<commit_after>/*
Copyright IBM Corp. 2017 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

                 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package blocksprovider

import (
	"math"
	"time"

	"sync/atomic"

	"github.com/golang/protobuf/proto"
	gossipcommon "github.com/hyperledger/fabric/gossip/common"
	"github.com/hyperledger/fabric/gossip/discovery"

	"github.com/hyperledger/fabric/common/flogging"
	"github.com/hyperledger/fabric/gossip/api"
	"github.com/hyperledger/fabric/protos/common"
	gossip_proto "github.com/hyperledger/fabric/protos/gossip"
	"github.com/hyperledger/fabric/protos/orderer"
	"github.com/op/go-logging"
)

// LedgerInfo an adapter to provide the interface to query
// the ledger committer for current ledger height
type LedgerInfo interface {
	// LedgerHeight returns current local ledger height
	LedgerHeight() (uint64, error)
}

// GossipServiceAdapter serves to provide basic functionality
// required from gossip service by delivery service
type GossipServiceAdapter interface {
	// PeersOfChannel returns slice with members of specified channel
	PeersOfChannel(gossipcommon.ChainID) []discovery.NetworkMember

	// AddPayload adds payload to the local state sync buffer
	AddPayload(chainID string, payload *gossip_proto.Payload) error

	// Gossip the message across the peers
	Gossip(msg *gossip_proto.GossipMessage)
}

// BlocksProvider used to read blocks from the ordering service
// for specified chain it subscribed to
type BlocksProvider interface {
	// DeliverBlocks starts delivering and disseminating blocks
	DeliverBlocks()

	// Stop shutdowns blocks provider and stops delivering new blocks
	Stop()
}

// BlocksDeliverer defines interface which actually helps
// to abstract the AtomicBroadcast_DeliverClient with only
// required method for blocks provider.
// This also decouples the production implementation of the gRPC stream
// from the code in order for the code to be more modular and testable.
type BlocksDeliverer interface {
	// Recv retrieves a response from the ordering service
	Recv() (*orderer.DeliverResponse, error)

	// Send sends an envelope to the ordering service
	Send(*common.Envelope) error
}

type streamClient interface {
	BlocksDeliverer

	// Close closes the stream and its underlying connection
	Close()

	// Disconnect disconnects from the remote node
	Disconnect()
}

// blocksProviderImpl the actual implementation for BlocksProvider interface
type blocksProviderImpl struct {
	chainID string

	client streamClient

	gossip GossipServiceAdapter

	mcs api.MessageCryptoService

	done int32

	wrongStatusThreshold int
}

const wrongStatusThreshold = 10

var maxRetryDelay = time.Second * 10

var logger *logging.Logger // package-level logger

func init() {
	logger = flogging.MustGetLogger("blocksProvider")
}

// NewBlocksProvider constructor function to create blocks deliverer instance
func NewBlocksProvider(chainID string, client streamClient, gossip GossipServiceAdapter, mcs api.MessageCryptoService) BlocksProvider {
	return &blocksProviderImpl{
		chainID:              chainID,
		client:               client,
		gossip:               gossip,
		mcs:                  mcs,
		wrongStatusThreshold: wrongStatusThreshold,
	}
}

// DeliverBlocks used to pull out blocks from the ordering service to
// distributed them across peers
func (b *blocksProviderImpl) DeliverBlocks() {
	errorStatusCounter := 0
	statusCounter := 0
	defer b.client.Close()
	for !b.isDone() {
		msg, err := b.client.Recv()
		if err != nil {
			logger.Warningf("[%s] Receive error: %s", b.chainID, err.Error())
			return
		}
		switch t := msg.Type.(type) {
		case *orderer.DeliverResponse_Status:
			if t.Status == common.Status_SUCCESS {
				logger.Warningf("[%s] ERROR! Received success for a seek that should never complete", b.chainID)
				return
			}
			if t.Status == common.Status_BAD_REQUEST || t.Status == common.Status_FORBIDDEN {
				logger.Errorf("[%s] Got error %v", b.chainID, t)
				errorStatusCounter++
				if errorStatusCounter > b.wrongStatusThreshold {
					logger.Criticalf("[%s] Wrong statuses threshold passed, stopping block provider", b.chainID)
					return
				}
			} else {
				errorStatusCounter = 0
				logger.Warningf("[%s] Got error %v", b.chainID, t)
			}
			maxDelay := float64(maxRetryDelay)
			currDelay := float64(time.Duration(math.Pow(2, float64(statusCounter))) * 100 * time.Millisecond)
			time.Sleep(time.Duration(math.Min(maxDelay, currDelay)))
			if currDelay < maxDelay {
				statusCounter++
			}
			b.client.Disconnect()
			continue
		case *orderer.DeliverResponse_Block:
			errorStatusCounter = 0
			statusCounter = 0
			seqNum := t.Block.Header.Number

			marshaledBlock, err := proto.Marshal(t.Block)
			if err != nil {
				logger.Errorf("[%s] Error serializing block with sequence number %d, due to %s", b.chainID, seqNum, err)
				continue
			}
			if err := b.mcs.VerifyBlock(gossipcommon.ChainID(b.chainID), seqNum, marshaledBlock); err != nil {
				logger.Errorf("[%s] Error verifying block with sequnce number %d, due to %s", b.chainID, seqNum, err)
				continue
			}

			numberOfPeers := len(b.gossip.PeersOfChannel(gossipcommon.ChainID(b.chainID)))
			// Create payload with a block received
			payload := createPayload(seqNum, marshaledBlock)
			// Use payload to create gossip message
			gossipMsg := createGossipMsg(b.chainID, payload)

			logger.Debugf("[%s] Adding payload locally, buffer seqNum = [%d], peers number [%d]", b.chainID, seqNum, numberOfPeers)
			// Add payload to local state payloads buffer
			b.gossip.AddPayload(b.chainID, payload)

			// Gossip messages with other nodes
			logger.Debugf("[%s] Gossiping block [%d], peers number [%d]", b.chainID, seqNum, numberOfPeers)
			b.gossip.Gossip(gossipMsg)
		default:
			logger.Warningf("[%s] Received unknown: ", b.chainID, t)
			return
		}
	}
}

// Stop stops blocks delivery provider
func (b *blocksProviderImpl) Stop() {
	atomic.StoreInt32(&b.done, 1)
	b.client.Close()
}

// Check whenever provider is stopped
func (b *blocksProviderImpl) isDone() bool {
	return atomic.LoadInt32(&b.done) == 1
}

func createGossipMsg(chainID string, payload *gossip_proto.Payload) *gossip_proto.GossipMessage {
	gossipMsg := &gossip_proto.GossipMessage{
		Nonce:   0,
		Tag:     gossip_proto.GossipMessage_CHAN_AND_ORG,
		Channel: []byte(chainID),
		Content: &gossip_proto.GossipMessage_DataMsg{
			DataMsg: &gossip_proto.DataMessage{
				Payload: payload,
			},
		},
	}
	return gossipMsg
}

func createPayload(seqNum uint64, marshaledBlock []byte) *gossip_proto.Payload {
	return &gossip_proto.Payload{
		Data:   marshaledBlock,
		SeqNum: seqNum,
	}
}
<|endoftext|>