<commit_before>package stager_test

import (
	"encoding/json"
	"errors"
	"time"

	"github.com/cloudfoundry/storeadapter"

	"github.com/cloudfoundry-incubator/runtime-schema/bbs/fake_bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	. "github.com/cloudfoundry-incubator/stager/stager"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Stage", func() {
	var (
		stager                        Stager
		bbs                           *fake_bbs.FakeStagerBBS
		stagingRequest                models.StagingRequestFromCC
		downloadTailorAction          models.ExecutorAction
		downloadAppAction             models.ExecutorAction
		downloadFirstBuildpackAction  models.ExecutorAction
		downloadSecontBuildpackAction models.ExecutorAction
		downloadBuildArtifactsAction  models.ExecutorAction
		runAction                     models.ExecutorAction
		uploadDropletAction           models.ExecutorAction
		uploadBuildArtifactsAction    models.ExecutorAction
		fetchResultsAction            models.ExecutorAction
		config                        Config
	)

	BeforeEach(func() {
		bbs = &fake_bbs.FakeStagerBBS{}

		config = Config{
			Circuses: map[string]string{
				"penguin":                "penguin-compiler",
				"rabbit_hole":            "rabbit-hole-compiler",
				"compiler_with_full_url": "http://the-full-compiler-url",
				"compiler_with_bad_url":  "ftp://the-bad-compiler-url",
			},
			MinDiskMB:          2048,
			MinMemoryMB:        1024,
			MinFileDescriptors: 256,
		}

		stager = New(bbs, config)

		stagingRequest = models.StagingRequestFromCC{
			AppId:                          "bunny",
			TaskId:                         "hop",
			AppBitsDownloadUri:             "http://example-uri.com/bunny",
			BuildArtifactsCacheDownloadUri: "http://example-uri.com/bunny-droppings",
			Stack:           "rabbit_hole",
			FileDescriptors: 512,
			MemoryMB:        2048,
			DiskMB:          3072,
			Buildpacks: []models.Buildpack{
				{Name: "zfirst", Key: "zfirst-buildpack", Url: "first-buildpack-url"},
				{Name: "asecond", Key: "asecond-buildpack", Url: "second-buildpack-url"},
			},
			Environment: []models.EnvironmentVariable{
				{"VCAP_APPLICATION", "foo"},
				{"VCAP_SERVICES", "bar"},
			},
		}

		downloadTailorAction = models.EmitProgressFor(
			models.ExecutorAction{
				models.DownloadAction{
					From:     "http://file-server.com/v1/static/rabbit-hole-compiler",
					To:       "/tmp/circus",
					Extract:  true,
					CacheKey: "tailor-rabbit_hole",
				},
			},
			"",
			"",
			"Failed to Download Tailor",
		)

		downloadAppAction = models.EmitProgressFor(
			models.ExecutorAction{
				models.DownloadAction{
					From:    "http://example-uri.com/bunny",
					To:      "/app",
					Extract: true,
				},
			},
			"Downloading App Package",
			"Downloaded App Package",
			"Failed to Download App Package",
		)

		downloadFirstBuildpackAction = models.EmitProgressFor(
			models.ExecutorAction{
				models.DownloadAction{
					From:     "first-buildpack-url",
					To:       "/tmp/buildpacks/0fe7d5fc3f73b0ab8682a664da513fbd",
					Extract:  true,
					CacheKey: "zfirst-buildpack",
				},
			},
			"Downloading Buildpack: zfirst",
			"Downloaded Buildpack: zfirst",
			"Failed to Download Buildpack: zfirst",
		)

		downloadSecontBuildpackAction = models.EmitProgressFor(
			models.ExecutorAction{
				models.DownloadAction{
					From:     "second-buildpack-url",
					To:       "/tmp/buildpacks/58015c32d26f0ad3418f87dd9bf47797",
					Extract:  true,
					CacheKey: "asecond-buildpack",
				},
			},
			"Downloading Buildpack: asecond",
			"Downloaded Buildpack: asecond",
			"Failed to Download Buildpack: asecond",
		)

		downloadBuildArtifactsAction = models.Try(
			models.EmitProgressFor(
				models.ExecutorAction{
					models.DownloadAction{
						From:    "http://example-uri.com/bunny-droppings",
						To:      "/tmp/cache",
						Extract: true,
					},
				},
				"Downloading Build Artifacts Cache",
				"Downloaded Build Artifacts Cache",
				"No Build Artifacts Cache Found.  Proceeding...",
			),
		)

		fileDescriptorLimit := uint64(512)

		runAction = models.EmitProgressFor(
			models.ExecutorAction{
				models.RunAction{
					Path: "/tmp/circus/tailor",
					Args: []string{
						"-appDir=/app",
						"-buildArtifactsCacheDir=/tmp/cache",
						"-buildpackOrder=zfirst-buildpack,asecond-buildpack",
						"-buildpacksDir=/tmp/buildpacks",
						"-outputDropletDir=/tmp/droplet",
						"-outputMetadataDir=/tmp/result",
					},
					Env: []models.EnvironmentVariable{
						{"VCAP_APPLICATION", "foo"},
						{"VCAP_SERVICES", "bar"},
					},
					Timeout:        15 * time.Minute,
					ResourceLimits: models.ResourceLimits{Nofile: &fileDescriptorLimit},
				},
			},
			"Staging...",
			"Staging Complete",
			"Staging Failed",
		)

		uploadDropletAction = models.EmitProgressFor(
			models.ExecutorAction{
				models.UploadAction{
					From:     "/tmp/droplet/",
					To:       "http://file-server.com/v1/droplet/bunny",
					Compress: false,
				},
			},
			"Uploading Droplet",
			"Droplet Uploaded",
			"Failed to Upload Droplet",
		)

		uploadBuildArtifactsAction = models.Try(
			models.EmitProgressFor(
				models.ExecutorAction{
					models.UploadAction{
						From:     "/tmp/cache/",
						To:       "http://file-server.com/v1/build_artifacts/bunny",
						Compress: true,
					},
				},
				"Uploading Build Artifacts Cache",
				"Uploaded Build Artifacts Cache",
				"Failed to Upload Build Artifacts Cache.  Proceeding...",
			),
		)

		fetchResultsAction = models.EmitProgressFor(
			models.ExecutorAction{
				models.FetchResultAction{
					File: "/tmp/result/result.json",
				},
			},
			"",
			"",
			"Failed to Fetch Detected Buildpack",
		)
	})

	Context("when file the server is available", func() {
		BeforeEach(func() {
			bbs.GetAvailableFileServerReturns("http://file-server.com/", nil)
		})

		It("creates a Task with staging instructions", func() {
			err := stager.Stage(stagingRequest)
			Ω(err).ShouldNot(HaveOccurred())

			desiredTask := bbs.DesireTaskArgsForCall(0)

			Ω(desiredTask.Guid).To(Equal("bunny-hop"))
			Ω(desiredTask.Stack).To(Equal("rabbit_hole"))
			Ω(desiredTask.Log.Guid).To(Equal("bunny"))
			Ω(desiredTask.Log.SourceName).To(Equal("STG"))
			Ω(desiredTask.Log.Index).To(BeNil())

			var annotation models.StagingTaskAnnotation

			err = json.Unmarshal([]byte(desiredTask.Annotation), &annotation)
			Ω(err).ShouldNot(HaveOccurred())

			Ω(annotation).Should(Equal(models.StagingTaskAnnotation{
				AppId:  "bunny",
				TaskId: "hop",
			}))

			Ω(desiredTask.Actions).Should(Equal([]models.ExecutorAction{
				downloadTailorAction,
				downloadAppAction,
				downloadFirstBuildpackAction,
				downloadSecontBuildpackAction,
				downloadBuildArtifactsAction,
				runAction,
				uploadDropletAction,
				uploadBuildArtifactsAction,
				fetchResultsAction,
			}))

			Ω(desiredTask.MemoryMB).To(Equal(2048))
			Ω(desiredTask.DiskMB).To(Equal(3072))
		})

		Describe("resource limits", func() {
			Context("when the app's memory limit is less than the minimum memory", func() {
				BeforeEach(func() {
					stagingRequest.MemoryMB = 256
				})

				It("uses the minimum memory", func() {
					err := stager.Stage(stagingRequest)
					Ω(err).ShouldNot(HaveOccurred())

					desiredTask := bbs.DesireTaskArgsForCall(0)
					Ω(desiredTask.MemoryMB).Should(BeNumerically("==", config.MinMemoryMB))
				})
			})

			Context("when the app's disk limit is less than the minimum disk", func() {
				BeforeEach(func() {
					stagingRequest.DiskMB = 256
				})

				It("uses the minimum disk", func() {
					err := stager.Stage(stagingRequest)
					Ω(err).ShouldNot(HaveOccurred())

					desiredTask := bbs.DesireTaskArgsForCall(0)
					Ω(desiredTask.DiskMB).Should(BeNumerically("==", config.MinDiskMB))
				})
			})

			Context("when the app's memory limit is less than the minimum memory", func() {
				BeforeEach(func() {
					stagingRequest.FileDescriptors = 17
				})

				It("uses the minimum file descriptors", func() {
					err := stager.Stage(stagingRequest)
					Ω(err).ShouldNot(HaveOccurred())

					desiredTask := bbs.DesireTaskArgsForCall(0)

					runAction = models.EmitProgressFor(
						models.ExecutorAction{
							models.RunAction{
								Path: "/tmp/circus/tailor",
								Args: []string{
									"-appDir=/app",
									"-buildArtifactsCacheDir=/tmp/cache",
									"-buildpackOrder=zfirst-buildpack,asecond-buildpack",
									"-buildpacksDir=/tmp/buildpacks",
									"-outputDropletDir=/tmp/droplet",
									"-outputMetadataDir=/tmp/result",
								},
								Env: []models.EnvironmentVariable{
									{"VCAP_APPLICATION", "foo"},
									{"VCAP_SERVICES", "bar"},
								},
								Timeout:        15 * time.Minute,
								ResourceLimits: models.ResourceLimits{Nofile: &config.MinFileDescriptors},
							},
						},
						"Staging...",
						"Staging Complete",
						"Staging Failed",
					)

					Ω(desiredTask.Actions).Should(Equal([]models.ExecutorAction{
						downloadTailorAction,
						downloadAppAction,
						downloadFirstBuildpackAction,
						downloadSecontBuildpackAction,
						downloadBuildArtifactsAction,
						runAction,
						uploadDropletAction,
						uploadBuildArtifactsAction,
						fetchResultsAction,
					}))
				})
			})
		})

		Context("when build artifacts download uris are not provided", func() {
			BeforeEach(func() {
				stagingRequest.BuildArtifactsCacheDownloadUri = ""
			})

			It("does not instruct the executor to download the cache", func() {
				err := stager.Stage(stagingRequest)
				Ω(err).ShouldNot(HaveOccurred())

				desiredTask := bbs.DesireTaskArgsForCall(0)

				Ω(desiredTask.Actions).Should(Equal([]models.ExecutorAction{
					downloadTailorAction,
					downloadAppAction,
					downloadFirstBuildpackAction,
					downloadSecontBuildpackAction,
					runAction,
					uploadDropletAction,
					uploadBuildArtifactsAction,
					fetchResultsAction,
				}))
			})
		})

		Context("when no compiler is defined for the requested stack in stager configuration", func() {
			BeforeEach(func() {
				stagingRequest.Stack = "no_such_stack"
			})

			It("returns an error", func() {
				err := stager.Stage(stagingRequest)

				Ω(err).Should(HaveOccurred())
				Ω(err.Error()).Should(Equal("no compiler defined for requested stack"))
			})
		})

		Context("when the compiler for the requested stack is specified as a full URL", func() {
			BeforeEach(func() {
				stagingRequest.Stack = "compiler_with_full_url"
			})

			It("uses the full URL in the download tailor action", func() {
				err := stager.Stage(stagingRequest)
				Ω(err).ShouldNot(HaveOccurred())

				desiredTask := bbs.DesireTaskArgsForCall(0)

				downloadAction := desiredTask.Actions[0].Action.(models.EmitProgressAction).Action.Action.(models.DownloadAction)
				Ω(downloadAction.From).Should(Equal("http://the-full-compiler-url"))
			})
		})

		Context("when the compiler for the requested stack is specified as a full URL with an unexpected scheme", func() {
			BeforeEach(func() {
				stagingRequest.Stack = "compiler_with_bad_url"
			})

			It("returns an error", func() {
				err := stager.Stage(stagingRequest)
				Ω(err).Should(HaveOccurred())
			})
		})

		Context("when build artifacts download url is not a valid url", func() {
			BeforeEach(func() {
				stagingRequest.BuildArtifactsCacheDownloadUri = "not-a-uri"
			})

			It("return a url parsing error", func() {
				err := stager.Stage(stagingRequest)

				Ω(err).Should(HaveOccurred())
				Ω(err.Error()).Should(ContainSubstring("invalid URI"))
			})
		})

		Context("when the task has already been created", func() {
			BeforeEach(func() {
				bbs.DesireTaskReturns(storeadapter.ErrorKeyExists)
			})

			It("does not raise an error", func() {
				err := stager.Stage(stagingRequest)
				Ω(err).ShouldNot(HaveOccurred())
			})
		})

		Context("when writing the task to the BBS fails", func() {
			desireErr := errors.New("Could not connect!")

			BeforeEach(func() {
				bbs.DesireTaskReturns(desireErr)
			})

			It("returns an error", func() {
				err := stager.Stage(stagingRequest)
				Ω(err).Should(Equal(desireErr))
			})
		})
	})

	Context("when file server is not available", func() {
		BeforeEach(func() {
			bbs.GetAvailableFileServerReturns("http://file-server.com/", storeadapter.ErrorKeyNotFound)
		})

		It("should return an error", func() {
			err := stager.Stage(models.StagingRequestFromCC{
				AppId:                          "bunny",
				TaskId:                         "hop",
				AppBitsDownloadUri:             "http://example-uri.com/bunny",
				BuildArtifactsCacheDownloadUri: "http://example-uri.com/bunny-droppings",
				Stack:    "rabbit_hole",
				MemoryMB: 256,
				DiskMB:   1024,
			})

			Ω(err).Should(HaveOccurred())
			Ω(err.Error()).Should(Equal("no available file server present"))
		})
	})
})
<commit_msg>Fix stager_test<commit_after>package stager_test

import (
	"encoding/json"
	"errors"
	"time"

	"github.com/cloudfoundry/gosteno"
	"github.com/cloudfoundry/storeadapter"

	"github.com/cloudfoundry-incubator/runtime-schema/bbs/fake_bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	. "github.com/cloudfoundry-incubator/stager/stager"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Stage", func() {
	var (
		stager                        Stager
		bbs                           *fake_bbs.FakeStagerBBS
		stagingRequest                models.StagingRequestFromCC
		downloadTailorAction          models.ExecutorAction
		downloadAppAction             models.ExecutorAction
		downloadFirstBuildpackAction  models.ExecutorAction
		downloadSecontBuildpackAction models.ExecutorAction
		downloadBuildArtifactsAction  models.ExecutorAction
		runAction                     models.ExecutorAction
		uploadDropletAction           models.ExecutorAction
		uploadBuildArtifactsAction    models.ExecutorAction
		fetchResultsAction            models.ExecutorAction
		config                        Config
	)

	BeforeEach(func() {
		bbs = &fake_bbs.FakeStagerBBS{}
		logger := gosteno.NewLogger("stager-test")

		config = Config{
			Circuses: map[string]string{
				"penguin":                "penguin-compiler",
				"rabbit_hole":            "rabbit-hole-compiler",
				"compiler_with_full_url": "http://the-full-compiler-url",
				"compiler_with_bad_url":  "ftp://the-bad-compiler-url",
			},
			MinDiskMB:          2048,
			MinMemoryMB:        1024,
			MinFileDescriptors: 256,
		}

		stager = New(bbs, logger, config)

		stagingRequest = models.StagingRequestFromCC{
			AppId:                          "bunny",
			TaskId:                         "hop",
			AppBitsDownloadUri:             "http://example-uri.com/bunny",
			BuildArtifactsCacheDownloadUri: "http://example-uri.com/bunny-droppings",
			Stack:           "rabbit_hole",
			FileDescriptors: 512,
			MemoryMB:        2048,
			DiskMB:          3072,
			Buildpacks: []models.Buildpack{
				{Name: "zfirst", Key: "zfirst-buildpack", Url: "first-buildpack-url"},
				{Name: "asecond", Key: "asecond-buildpack", Url: "second-buildpack-url"},
			},
			Environment: []models.EnvironmentVariable{
				{"VCAP_APPLICATION", "foo"},
				{"VCAP_SERVICES", "bar"},
			},
		}

		downloadTailorAction = models.EmitProgressFor(
			models.ExecutorAction{
				models.DownloadAction{
					From:     "http://file-server.com/v1/static/rabbit-hole-compiler",
					To:       "/tmp/circus",
					Extract:  true,
					CacheKey: "tailor-rabbit_hole",
				},
			},
			"",
			"",
			"Failed to Download Tailor",
		)

		downloadAppAction = models.EmitProgressFor(
			models.ExecutorAction{
				models.DownloadAction{
					From:    "http://example-uri.com/bunny",
					To:      "/app",
					Extract: true,
				},
			},
			"Downloading App Package",
			"Downloaded App Package",
			"Failed to Download App Package",
		)

		downloadFirstBuildpackAction = models.EmitProgressFor(
			models.ExecutorAction{
				models.DownloadAction{
					From:     "first-buildpack-url",
					To:       "/tmp/buildpacks/0fe7d5fc3f73b0ab8682a664da513fbd",
					Extract:  true,
					CacheKey: "zfirst-buildpack",
				},
			},
			"Downloading Buildpack: zfirst",
			"Downloaded Buildpack: zfirst",
			"Failed to Download Buildpack: zfirst",
		)

		downloadSecontBuildpackAction = models.EmitProgressFor(
			models.ExecutorAction{
				models.DownloadAction{
					From:     "second-buildpack-url",
					To:       "/tmp/buildpacks/58015c32d26f0ad3418f87dd9bf47797",
					Extract:  true,
					CacheKey: "asecond-buildpack",
				},
			},
			"Downloading Buildpack: asecond",
			"Downloaded Buildpack: asecond",
			"Failed to Download Buildpack: asecond",
		)

		downloadBuildArtifactsAction = models.Try(
			models.EmitProgressFor(
				models.ExecutorAction{
					models.DownloadAction{
						From:    "http://example-uri.com/bunny-droppings",
						To:      "/tmp/cache",
						Extract: true,
					},
				},
				"Downloading Build Artifacts Cache",
				"Downloaded Build Artifacts Cache",
				"No Build Artifacts Cache Found.  Proceeding...",
			),
		)

		fileDescriptorLimit := uint64(512)

		runAction = models.EmitProgressFor(
			models.ExecutorAction{
				models.RunAction{
					Path: "/tmp/circus/tailor",
					Args: []string{
						"-appDir=/app",
						"-buildArtifactsCacheDir=/tmp/cache",
						"-buildpackOrder=zfirst-buildpack,asecond-buildpack",
						"-buildpacksDir=/tmp/buildpacks",
						"-outputDropletDir=/tmp/droplet",
						"-outputMetadataDir=/tmp/result",
					},
					Env: []models.EnvironmentVariable{
						{"VCAP_APPLICATION", "foo"},
						{"VCAP_SERVICES", "bar"},
					},
					Timeout:        15 * time.Minute,
					ResourceLimits: models.ResourceLimits{Nofile: &fileDescriptorLimit},
				},
			},
			"Staging...",
			"Staging Complete",
			"Staging Failed",
		)

		uploadDropletAction = models.EmitProgressFor(
			models.ExecutorAction{
				models.UploadAction{
					From:     "/tmp/droplet/",
					To:       "http://file-server.com/v1/droplet/bunny",
					Compress: false,
				},
			},
			"Uploading Droplet",
			"Droplet Uploaded",
			"Failed to Upload Droplet",
		)

		uploadBuildArtifactsAction = models.Try(
			models.EmitProgressFor(
				models.ExecutorAction{
					models.UploadAction{
						From:     "/tmp/cache/",
						To:       "http://file-server.com/v1/build_artifacts/bunny",
						Compress: true,
					},
				},
				"Uploading Build Artifacts Cache",
				"Uploaded Build Artifacts Cache",
				"Failed to Upload Build Artifacts Cache.  Proceeding...",
			),
		)

		fetchResultsAction = models.EmitProgressFor(
			models.ExecutorAction{
				models.FetchResultAction{
					File: "/tmp/result/result.json",
				},
			},
			"",
			"",
			"Failed to Fetch Detected Buildpack",
		)
	})

	Context("when file the server is available", func() {
		BeforeEach(func() {
			bbs.GetAvailableFileServerReturns("http://file-server.com/", nil)
		})

		It("creates a Task with staging instructions", func() {
			err := stager.Stage(stagingRequest)
			Ω(err).ShouldNot(HaveOccurred())

			desiredTask := bbs.DesireTaskArgsForCall(0)

			Ω(desiredTask.Guid).To(Equal("bunny-hop"))
			Ω(desiredTask.Stack).To(Equal("rabbit_hole"))
			Ω(desiredTask.Log.Guid).To(Equal("bunny"))
			Ω(desiredTask.Log.SourceName).To(Equal("STG"))
			Ω(desiredTask.Log.Index).To(BeNil())

			var annotation models.StagingTaskAnnotation

			err = json.Unmarshal([]byte(desiredTask.Annotation), &annotation)
			Ω(err).ShouldNot(HaveOccurred())

			Ω(annotation).Should(Equal(models.StagingTaskAnnotation{
				AppId:  "bunny",
				TaskId: "hop",
			}))

			Ω(desiredTask.Actions).Should(Equal([]models.ExecutorAction{
				downloadTailorAction,
				downloadAppAction,
				downloadFirstBuildpackAction,
				downloadSecontBuildpackAction,
				downloadBuildArtifactsAction,
				runAction,
				uploadDropletAction,
				uploadBuildArtifactsAction,
				fetchResultsAction,
			}))

			Ω(desiredTask.MemoryMB).To(Equal(2048))
			Ω(desiredTask.DiskMB).To(Equal(3072))
		})

		Describe("resource limits", func() {
			Context("when the app's memory limit is less than the minimum memory", func() {
				BeforeEach(func() {
					stagingRequest.MemoryMB = 256
				})

				It("uses the minimum memory", func() {
					err := stager.Stage(stagingRequest)
					Ω(err).ShouldNot(HaveOccurred())

					desiredTask := bbs.DesireTaskArgsForCall(0)
					Ω(desiredTask.MemoryMB).Should(BeNumerically("==", config.MinMemoryMB))
				})
			})

			Context("when the app's disk limit is less than the minimum disk", func() {
				BeforeEach(func() {
					stagingRequest.DiskMB = 256
				})

				It("uses the minimum disk", func() {
					err := stager.Stage(stagingRequest)
					Ω(err).ShouldNot(HaveOccurred())

					desiredTask := bbs.DesireTaskArgsForCall(0)
					Ω(desiredTask.DiskMB).Should(BeNumerically("==", config.MinDiskMB))
				})
			})

			Context("when the app's memory limit is less than the minimum memory", func() {
				BeforeEach(func() {
					stagingRequest.FileDescriptors = 17
				})

				It("uses the minimum file descriptors", func() {
					err := stager.Stage(stagingRequest)
					Ω(err).ShouldNot(HaveOccurred())

					desiredTask := bbs.DesireTaskArgsForCall(0)

					runAction = models.EmitProgressFor(
						models.ExecutorAction{
							models.RunAction{
								Path: "/tmp/circus/tailor",
								Args: []string{
									"-appDir=/app",
									"-buildArtifactsCacheDir=/tmp/cache",
									"-buildpackOrder=zfirst-buildpack,asecond-buildpack",
									"-buildpacksDir=/tmp/buildpacks",
									"-outputDropletDir=/tmp/droplet",
									"-outputMetadataDir=/tmp/result",
								},
								Env: []models.EnvironmentVariable{
									{"VCAP_APPLICATION", "foo"},
									{"VCAP_SERVICES", "bar"},
								},
								Timeout:        15 * time.Minute,
								ResourceLimits: models.ResourceLimits{Nofile: &config.MinFileDescriptors},
							},
						},
						"Staging...",
						"Staging Complete",
						"Staging Failed",
					)

					Ω(desiredTask.Actions).Should(Equal([]models.ExecutorAction{
						downloadTailorAction,
						downloadAppAction,
						downloadFirstBuildpackAction,
						downloadSecontBuildpackAction,
						downloadBuildArtifactsAction,
						runAction,
						uploadDropletAction,
						uploadBuildArtifactsAction,
						fetchResultsAction,
					}))
				})
			})
		})

		Context("when build artifacts download uris are not provided", func() {
			BeforeEach(func() {
				stagingRequest.BuildArtifactsCacheDownloadUri = ""
			})

			It("does not instruct the executor to download the cache", func() {
				err := stager.Stage(stagingRequest)
				Ω(err).ShouldNot(HaveOccurred())

				desiredTask := bbs.DesireTaskArgsForCall(0)

				Ω(desiredTask.Actions).Should(Equal([]models.ExecutorAction{
					downloadTailorAction,
					downloadAppAction,
					downloadFirstBuildpackAction,
					downloadSecontBuildpackAction,
					runAction,
					uploadDropletAction,
					uploadBuildArtifactsAction,
					fetchResultsAction,
				}))
			})
		})

		Context("when no compiler is defined for the requested stack in stager configuration", func() {
			BeforeEach(func() {
				stagingRequest.Stack = "no_such_stack"
			})

			It("returns an error", func() {
				err := stager.Stage(stagingRequest)

				Ω(err).Should(HaveOccurred())
				Ω(err.Error()).Should(Equal("no compiler defined for requested stack"))
			})
		})

		Context("when the compiler for the requested stack is specified as a full URL", func() {
			BeforeEach(func() {
				stagingRequest.Stack = "compiler_with_full_url"
			})

			It("uses the full URL in the download tailor action", func() {
				err := stager.Stage(stagingRequest)
				Ω(err).ShouldNot(HaveOccurred())

				desiredTask := bbs.DesireTaskArgsForCall(0)

				downloadAction := desiredTask.Actions[0].Action.(models.EmitProgressAction).Action.Action.(models.DownloadAction)
				Ω(downloadAction.From).Should(Equal("http://the-full-compiler-url"))
			})
		})

		Context("when the compiler for the requested stack is specified as a full URL with an unexpected scheme", func() {
			BeforeEach(func() {
				stagingRequest.Stack = "compiler_with_bad_url"
			})

			It("returns an error", func() {
				err := stager.Stage(stagingRequest)
				Ω(err).Should(HaveOccurred())
			})
		})

		Context("when build artifacts download url is not a valid url", func() {
			BeforeEach(func() {
				stagingRequest.BuildArtifactsCacheDownloadUri = "not-a-uri"
			})

			It("return a url parsing error", func() {
				err := stager.Stage(stagingRequest)

				Ω(err).Should(HaveOccurred())
				Ω(err.Error()).Should(ContainSubstring("invalid URI"))
			})
		})

		Context("when the task has already been created", func() {
			BeforeEach(func() {
				bbs.DesireTaskReturns(storeadapter.ErrorKeyExists)
			})

			It("does not raise an error", func() {
				err := stager.Stage(stagingRequest)
				Ω(err).ShouldNot(HaveOccurred())
			})
		})

		Context("when writing the task to the BBS fails", func() {
			desireErr := errors.New("Could not connect!")

			BeforeEach(func() {
				bbs.DesireTaskReturns(desireErr)
			})

			It("returns an error", func() {
				err := stager.Stage(stagingRequest)
				Ω(err).Should(Equal(desireErr))
			})
		})
	})

	Context("when file server is not available", func() {
		BeforeEach(func() {
			bbs.GetAvailableFileServerReturns("http://file-server.com/", storeadapter.ErrorKeyNotFound)
		})

		It("should return an error", func() {
			err := stager.Stage(models.StagingRequestFromCC{
				AppId:                          "bunny",
				TaskId:                         "hop",
				AppBitsDownloadUri:             "http://example-uri.com/bunny",
				BuildArtifactsCacheDownloadUri: "http://example-uri.com/bunny-droppings",
				Stack:    "rabbit_hole",
				MemoryMB: 256,
				DiskMB:   1024,
			})

			Ω(err).Should(HaveOccurred())
			Ω(err.Error()).Should(Equal("no available file server present"))
		})
	})
})
<|endoftext|>


<commit_before>package statements

const (
	// Databases list all data bases
	DatabasesSelect = `
SELECT
	datname
FROM
	pg_database`
	DatabasesWhere = `
WHERE
	NOT datistemplate`
	DatabasesOrderBy = `
ORDER BY
	datname ASC`
	Databases = DatabasesSelect + DatabasesWhere + DatabasesOrderBy
	// Schemas list all schema on data base
	Schemas = `
SELECT
	schema_name
FROM
	information_schema.schemata
ORDER BY
	schema_name ASC`

	// Tables list all tables
	TablesSelect = `
SELECT
	n.nspname as "schema",
	c.relname as "name",
	CASE c.relkind
		WHEN 'r' THEN 'table'
		WHEN 'v' THEN 'view'
		WHEN 'm' THEN 'materialized_view'
		WHEN 'i' THEN 'index'
		WHEN 'S' THEN 'sequence'
		WHEN 's' THEN 'special'
		WHEN 'f' THEN 'foreign_table'
	END as "type",
	pg_catalog.pg_get_userbyid(c.relowner) as "owner"
FROM
	pg_catalog.pg_class c
LEFT JOIN
	pg_catalog.pg_namespace n ON n.oid = c.relnamespace `
	TablesWhere = `
WHERE
	c.relkind IN ('r','v','m','S','s','') AND
	n.nspname !~ '^pg_toast' AND
	n.nspname NOT IN ('information_schema', 'pg_catalog') AND
	has_schema_privilege(n.nspname, 'USAGE') `
	TablesOrderBy = `
ORDER BY 1, 2`
	Tables = TablesSelect + TablesWhere + TablesOrderBy
)
<commit_msg>splits schemas listing sql in conditions<commit_after>package statements

const (
	// Databases list all data bases
	DatabasesSelect = `
SELECT
	datname
FROM
	pg_database`
	DatabasesWhere = `
WHERE
	NOT datistemplate`
	DatabasesOrderBy = `
ORDER BY
	datname ASC`
	Databases = DatabasesSelect + DatabasesWhere + DatabasesOrderBy
	// Schemas list all schema on data base
	SchemasSelect = `
SELECT
	schema_name
FROM
	information_schema.schemata`
	SchemasOrderBy = `
ORDER BY
	schema_name ASC`
	Schemas = SchemasSelect + SchemasOrderBy

	// Tables list all tables
	TablesSelect = `
SELECT
	n.nspname as "schema",
	c.relname as "name",
	CASE c.relkind
		WHEN 'r' THEN 'table'
		WHEN 'v' THEN 'view'
		WHEN 'm' THEN 'materialized_view'
		WHEN 'i' THEN 'index'
		WHEN 'S' THEN 'sequence'
		WHEN 's' THEN 'special'
		WHEN 'f' THEN 'foreign_table'
	END as "type",
	pg_catalog.pg_get_userbyid(c.relowner) as "owner"
FROM
	pg_catalog.pg_class c
LEFT JOIN
	pg_catalog.pg_namespace n ON n.oid = c.relnamespace `
	TablesWhere = `
WHERE
	c.relkind IN ('r','v','m','S','s','') AND
	n.nspname !~ '^pg_toast' AND
	n.nspname NOT IN ('information_schema', 'pg_catalog') AND
	has_schema_privilege(n.nspname, 'USAGE') `
	TablesOrderBy = `
ORDER BY 1, 2`
	Tables = TablesSelect + TablesWhere + TablesOrderBy
)
<|endoftext|>


<commit_before>package awsecs

import (
	"reflect"
	"testing"
)

const (
	testCluster = "test-cluster"
	testFamily = "test-family"
	testTaskARN = "arn:aws:ecs:us-east-1:123456789012:task/12345678-9abc-def0-1234-56789abcdef0"
	testContainer = "test-container"
	testContainerData = map[string]string{
		docker.LabelPrefix + "com.amazonaws.ecs.task-arn":
			testTaskARN,
		docker.LabelPrefix + "com.amazonaws.ecs.cluster":
			testCluster,
		docker.LabelPrefix + "com.amazonaws.ecs.task-definition-family":
			testFamily,
	}
)

func getTestContainerNode() report.Node {
	return report.MakeNodeWith(
		report.MakeContainerNodeID("test-container"),
		testContainerData
	)
}

func TestGetLabelInfo(t *testing.T) {
	r := Make()
	rpt, err := r.Report()
	if err != nil {
		t.Fatal("Error making report", err)
	}
	labelInfo := r.getLabelInfo(rpt)
	expected := map[string]map[string]*taskLabelInfo{}
	if !reflect.DeepEqual(labelInfo, expected) {
		t.Error("Empty report did not produce empty label info: %v != %v", labelInfo, expected)
	}

	rpt.Containers = rpt.Containers.AddNode(getTestContainerNode())
	labelInfo = r.getLabelInfo(rpt)
	expected = map[string]map[string]*taskLabelInfo{
		testCluster: map[string]*taskLabelInfo{
			testTaskARN: &taskLabelInfo{
				containerIDs: []string{testContainer},
				family: testFamily,
			}
		}
	}
	if !reflect.DeepEqual(labelInfo, expected) {
		t.Error("Did not get expected label info: %v != %v", labelInfo, expected)
	}
}

// Implements ecsClient
type mockEcsClient {
	t *testing.T
	expectedARNs []string
	info ecsInfo
}

func newMockEcsClient(t *testing.T, expectedARNs []string, info ecsInfo) *ecsClient {
	return &mockEcsClient{
		t,
		expectedARNs,
		info,
	}
}

func (c mockEcsClient) getInfo(taskARNs []string) ecsInfo {
	if !reflect.DeepEqual(taskARNs, c.expectedARNs) {
		c.t.Fatal("getInfo called with wrong ARNs: %v != %v", taskARNs, c.expectedARNs)
	}
	return c.info
}

func TestTagReport(t *testing.T) {
	r := Make()

	r.clientsByCluster[testCluster] = newMockEcsClient(
		t,
		[]string{},
		ecsInfo{
			// TODO fill in values below
			tasks: map[string]ecsTask{},
			services: map[string]ecsService{},
			taskServiceMap: map[string]string{},
		},
	)

	rpt, err := r.Report()
	if err != nil {
		t.Fatal("Error making report")
	}
	rpt = r.Tag(rpt)
	// TODO check it matches
}
<commit_msg>ecs probe: add tests for reporter<commit_after>package awsecs

import (
	"reflect"
	"testing"
	"time"

	"github.com/weaveworks/scope/probe/docker"
	"github.com/weaveworks/scope/report"
)

var (
	testCluster = "test-cluster"
	testFamily = "test-family"
	testTaskARN = "arn:aws:ecs:us-east-1:123456789012:task/12345678-9abc-def0-1234-56789abcdef0"
	testTaskCreatedAt = time.Unix(1483228800, 0)
	testTaskDefinitionARN = "arn:aws:ecs:us-east-1:123456789012:task-definition/deadbeef-dead-beef-dead-beefdeadbeef"
	testTaskStartedAt = time.Unix(1483228805, 0)
	testDeploymentID = "ecs-svc/1121123211234321"
	testServiceName = "test-service"
	testServiceCount = 1
	testContainer = "test-container"
	testContainerData = map[string]string{
		docker.LabelPrefix + "com.amazonaws.ecs.task-arn":
			testTaskARN,
		docker.LabelPrefix + "com.amazonaws.ecs.cluster":
			testCluster,
		docker.LabelPrefix + "com.amazonaws.ecs.task-definition-family":
			testFamily,
	}
)

func getTestContainerNode() report.Node {
	return report.MakeNodeWith(
		report.MakeContainerNodeID(testContainer),
		testContainerData,
	)
}

func TestGetLabelInfo(t *testing.T) {
	r := Make(1e6, time.Hour)
	rpt, err := r.Report()
	if err != nil {
		t.Fatalf("Error making report", err)
	}
	labelInfo := getLabelInfo(rpt)
	expected := map[string]map[string]*taskLabelInfo{}
	if !reflect.DeepEqual(labelInfo, expected) {
		t.Errorf("Empty report did not produce empty label info: %v != %v", labelInfo, expected)
	}

	rpt.Container = rpt.Container.AddNode(getTestContainerNode())
	labelInfo = getLabelInfo(rpt)
	expected = map[string]map[string]*taskLabelInfo{
		testCluster: map[string]*taskLabelInfo{
			testTaskARN: &taskLabelInfo{
				containerIDs: []string{report.MakeContainerNodeID(testContainer)},
				family: testFamily,
			},
		},
	}
	if !reflect.DeepEqual(labelInfo, expected) {
		t.Errorf("Did not get expected label info: %v != %v", labelInfo, expected)
	}
}

// Implements ecsClient
type mockEcsClient struct {
	t *testing.T
	expectedARNs []string
	info ecsInfo
}

func newMockEcsClient(t *testing.T, expectedARNs []string, info ecsInfo) ecsClient {
	return &mockEcsClient{
		t,
		expectedARNs,
		info,
	}
}

func (c mockEcsClient) getInfo(taskARNs []string) ecsInfo {
	if !reflect.DeepEqual(taskARNs, c.expectedARNs) {
		c.t.Fatalf("getInfo called with wrong ARNs: %v != %v", taskARNs, c.expectedARNs)
	}
	return c.info
}

func TestTagReport(t *testing.T) {
	r := Make(1e6, time.Hour)

	r.clientsByCluster[testCluster] = newMockEcsClient(
		t,
		[]string{testTaskARN},
		ecsInfo{
			tasks: map[string]ecsTask{
				testTaskARN: ecsTask{
					taskARN: testTaskARN,
					createdAt: testTaskCreatedAt,
					taskDefinitionARN: testTaskDefinitionARN,
					startedAt: testTaskStartedAt,
					startedBy: testDeploymentID,
				},
			},
			services: map[string]ecsService{
				testServiceName: ecsService{
					serviceName: testServiceName,
					deploymentIDs: []string{testDeploymentID},
					desiredCount: 1,
					pendingCount: 0,
					runningCount: 1,
					taskDefinitionARN: testTaskDefinitionARN,
				},
			},
			taskServiceMap: map[string]string{
				testTaskARN: testServiceName,
			},
		},
	)

	rpt, err := r.Report()
	if err != nil {
		t.Fatalf("Error making report")
	}
	rpt.Container = rpt.Container.AddNode(getTestContainerNode())
	rpt, err = r.Tag(rpt)
	if err != nil {
		t.Fatalf("Failed to tag: %v", err)
	}

	// Check task node is present and contains expected values
	task, ok := rpt.ECSTask.Nodes[report.MakeECSTaskNodeID(testTaskARN)]
	if !ok {
		t.Fatalf("Result report did not contain task %v: %v", testTaskARN, rpt.ECSTask.Nodes)
	}
	taskExpected := map[string]string{
		TaskFamily: testFamily,
		Cluster: testCluster,
		CreatedAt: testTaskCreatedAt.Format(time.RFC3339Nano),
	}
	for key, expectedValue := range taskExpected {
		value, ok := task.Latest.Lookup(key)
		if !ok {
			t.Errorf("Result task did not contain expected key %v: %v", key, task.Latest)
			continue
		}
		if value != expectedValue {
			t.Errorf("Result task did not contain expected value for key %v: %v != %v", key, value, expectedValue)
		}
	}

	// Check service node is present and contains expected values
	service, ok := rpt.ECSService.Nodes[report.MakeECSServiceNodeID(testServiceName)]
	if !ok {
		t.Fatalf("Result report did not contain service %v: %v", testServiceName, rpt.ECSService.Nodes)
	}
	serviceExpected := map[string]string{
		Cluster: testCluster,
		ServiceDesiredCount: "1",
		ServiceRunningCount: "1",
	}
	for key, expectedValue := range serviceExpected {
		value, ok := service.Latest.Lookup(key)
		if !ok {
			t.Errorf("Result service did not contain expected key %v: %v", key, service.Latest)
			continue
		}
		if value != expectedValue {
			t.Errorf("Result service did not contain expected value for key %v: %v != %v", key, value, expectedValue)
		}
	}

	// Check container node is present and contains expected parents
	container, ok := rpt.Container.Nodes[report.MakeContainerNodeID(testContainer)]
	if !ok {
		t.Fatalf("Result report did not contain container %v: %v", testContainer, rpt.Container.Nodes)
	}
	containerParentsExpected := map[string]string{
		report.ECSTask: report.MakeECSTaskNodeID(testTaskARN),
		report.ECSService: report.MakeECSServiceNodeID(testServiceName),
	}
	for key, expectedValue := range containerParentsExpected {
		values, ok := container.Parents.Lookup(key)
		if !ok {
			t.Errorf("Result container did not have any parents for key %v: %v", key, container.Parents)
		}
		if !values.Contains(expectedValue) {
			t.Errorf("Result container did not contain expected value %v as a parent for key %v: %v", expectedValue, key, values)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2014, Hǎiliàng Wáng. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cwrap

import (
	gcc "github.com/hailiang/go-gccxml"
	"io"
	"io/ioutil"
	"os"
	"path"
	"regexp"
	"runtime"
	"syscall"
)

var (
	GOPATH, _ = syscall.Getenv("GOPATH")
	OutputDir = GOPATH + "/src/"
)

type Header struct {
	Dir         string
	File        string
	NamePattern string
	OtherCode   string
	// Not define it in the package, but may still searchable as included types
	// because it may be manually defined.
	Excluded      []string
	CgoDirectives []string
	BoolTypes     []string
}

func (h Header) FullPath() string {
	file := h.Dir + h.File
	if !fileExists(file) {
		panic("Header file cannot be found: " + file)
	}
	return file
}

func (h Header) Write(w io.Writer) {
	fp(w, h.OtherCode)
	fp(w, "#include <", h.File, ">")
}

type Package struct {
	// Required
	PacName string
	PacPath string
	From    Header

	// Optional
	Included []*Package
	GoFile   string
	CFile    string
	HFile    string
	TypeRule map[string]string
	ArgRule  map[string]string

	// Internal
	pat         *regexp.Regexp
	localNames  map[string]string
	fileIds     SSet
	boolSet     SSet
	typeDeclMap TypeDeclMap
	Statistics
	*gcc.XmlDoc
}

func (pac *Package) Load() (err error) {
	if pac.From.NamePattern == "" {
		pac.From.NamePattern = ".*"
	}
	pac.pat = regexp.MustCompile(pac.From.NamePattern)
	pac.localNames = make(map[string]string)
	pac.initBoolSet()
	pac.typeDeclMap = make(TypeDeclMap)
	if err := pac.loadXmlDoc(); err != nil {
		return err
	}
	if err := pac.initFileIds(); err != nil {
		return err
	}
	for _, inc := range pac.Included {
		inc.XmlDoc = pac.XmlDoc
		if err := inc.Load(); err != nil {
			return err
		}
	}
	return nil
}

func (pac *Package) loadXmlDoc() error {
	if pac.XmlDoc != nil {
		return nil
	}
	f, err := ioutil.TempFile(".", "_cwrap-")
	if err != nil {
		return err
	}
	defer os.Remove(f.Name())
	for _, inc := range pac.Included {
		inc.From.Write(f)
	}
	pac.From.Write(f)
	f.Close()
	pac.XmlDoc, err = gcc.Xml{f.Name()}.Doc()
	//	pac.XmlDoc.Print()
	return err
}

func (pac *Package) initBoolSet() {
	pac.boolSet = NewSSet()
	for _, t := range pac.From.BoolTypes {
		pac.boolSet.Add(t)
	}
}

func (pac *Package) initFileIds() error {
	pac.fileIds = NewSSet()
	fnames, err := gcc.IncludeFiles(pac.From.FullPath())
	if err != nil {
		return err
	}
	for _, name := range fnames {
		for _, file := range pac.XmlDoc.Files {
			if file.CName() == name {
				pac.fileIds.Add(file.Id())
				break
			}
		}
	}
	return nil
}

func (pac *Package) goFile() string {
	if pac.GoFile != "" {
		return pac.GoFile
	}
	return pac.defaultFile() + ".go"
}

func (pac *Package) cFile() string {
	if pac.CFile != "" {
		return pac.CFile
	}
	return pac.defaultFile() + ".c"
}

func (pac *Package) hFile() string {
	if pac.HFile != "" {
		return pac.HFile
	}
	return pac.defaultFile() + ".h"
}

func (pac *Package) defaultFile() string {
	return OutputDir + pac.PacPath + "/auto_" + runtime.GOARCH
}

func (pac *Package) createFile(file string) (io.WriteCloser, error) {
	if err := os.MkdirAll(path.Dir(file), 0755); err != nil {
		return nil, err
	}
	f, err := os.Create(file)
	if err != nil {
		return nil, err
	}
	return f, nil
}

func (pac *Package) Wrap() error {
	g, err := pac.createFile(pac.goFile())
	if err != nil {
		return err
	}
	defer g.Close()
	c, err := pac.createFile(pac.cFile())
	if err != nil {
		return err
	}
	defer c.Close()
	h, err := pac.createFile(pac.hFile())
	if err != nil {
		return err
	}
	defer h.Close()
	if err := pac.prepare(); err != nil {
		return err
	}
	if err := pac.write(g, c, h); err != nil {
		return err
	}
	return gofmt(pac.goFile())
}

func (pac *Package) prepare() error {
	if pac.XmlDoc == nil {
		if err := pac.Load(); err != nil {
			return err
		}
	}
	// reset localNames
	pac.localNames = make(map[string]string)
	return nil
}

func (pac *Package) GenConst(file string) error {
	f, err := os.Create(file)
	if err != nil {
		return err
	}
	defer f.Close()
	ms, err := gcc.Xml{pac.From.FullPath()}.Macros()
	if err != nil {
		return err
	}
	consts := ms.Constants(pac.From.NamePattern)
	nm := make(map[string]string)
	for _, c := range consts {
		nm[c.Name] = upperName(c.Name, pac.pat)
	}

	fp(f, "package ", pac.PacName)
	fp(f, "")
	fp(f, "const (")
	for _, c := range consts {
		body := c.Body
		for k, v := range nm {
			body = replace(body, k, v)
		}
		fp(f, upperName(c.Name, pac.pat), "=", body)
	}
	fp(f, ")")
	return nil
}

type Statistics struct {
	DefCount int
}

func (s Statistics) Print() {
	p(s.DefCount, "declarations wrapped.")
}
<commit_msg>cwrap: Use the first dir in GOPATH for OutputDir when it is a list of directories.<commit_after>// Copyright 2014, Hǎiliàng Wáng. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cwrap

import (
	gcc "github.com/hailiang/go-gccxml"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
)

var (
	GOPATHs   = filepath.SplitList(os.Getenv("GOPATH"))
	OutputDir = GOPATHs[0] + "/src/"
)

type Header struct {
	Dir         string
	File        string
	NamePattern string
	OtherCode   string
	// Not define it in the package, but may still searchable as included types
	// because it may be manually defined.
	Excluded      []string
	CgoDirectives []string
	BoolTypes     []string
}

func (h Header) FullPath() string {
	file := h.Dir + h.File
	if !fileExists(file) {
		panic("Header file cannot be found: " + file)
	}
	return file
}

func (h Header) Write(w io.Writer) {
	fp(w, h.OtherCode)
	fp(w, "#include <", h.File, ">")
}

type Package struct {
	// Required
	PacName string
	PacPath string
	From    Header

	// Optional
	Included []*Package
	GoFile   string
	CFile    string
	HFile    string
	TypeRule map[string]string
	ArgRule  map[string]string

	// Internal
	pat         *regexp.Regexp
	localNames  map[string]string
	fileIds     SSet
	boolSet     SSet
	typeDeclMap TypeDeclMap
	Statistics
	*gcc.XmlDoc
}

func (pac *Package) Load() (err error) {
	if pac.From.NamePattern == "" {
		pac.From.NamePattern = ".*"
	}
	pac.pat = regexp.MustCompile(pac.From.NamePattern)
	pac.localNames = make(map[string]string)
	pac.initBoolSet()
	pac.typeDeclMap = make(TypeDeclMap)
	if err := pac.loadXmlDoc(); err != nil {
		return err
	}
	if err := pac.initFileIds(); err != nil {
		return err
	}
	for _, inc := range pac.Included {
		inc.XmlDoc = pac.XmlDoc
		if err := inc.Load(); err != nil {
			return err
		}
	}
	return nil
}

func (pac *Package) loadXmlDoc() error {
	if pac.XmlDoc != nil {
		return nil
	}
	f, err := ioutil.TempFile(".", "_cwrap-")
	if err != nil {
		return err
	}
	defer os.Remove(f.Name())
	for _, inc := range pac.Included {
		inc.From.Write(f)
	}
	pac.From.Write(f)
	f.Close()
	pac.XmlDoc, err = gcc.Xml{f.Name()}.Doc()
	//	pac.XmlDoc.Print()
	return err
}

func (pac *Package) initBoolSet() {
	pac.boolSet = NewSSet()
	for _, t := range pac.From.BoolTypes {
		pac.boolSet.Add(t)
	}
}

func (pac *Package) initFileIds() error {
	pac.fileIds = NewSSet()
	fnames, err := gcc.IncludeFiles(pac.From.FullPath())
	if err != nil {
		return err
	}
	for _, name := range fnames {
		for _, file := range pac.XmlDoc.Files {
			if file.CName() == name {
				pac.fileIds.Add(file.Id())
				break
			}
		}
	}
	return nil
}

func (pac *Package) goFile() string {
	if pac.GoFile != "" {
		return pac.GoFile
	}
	return pac.defaultFile() + ".go"
}

func (pac *Package) cFile() string {
	if pac.CFile != "" {
		return pac.CFile
	}
	return pac.defaultFile() + ".c"
}

func (pac *Package) hFile() string {
	if pac.HFile != "" {
		return pac.HFile
	}
	return pac.defaultFile() + ".h"
}

func (pac *Package) defaultFile() string {
	return OutputDir + pac.PacPath + "/auto_" + runtime.GOARCH
}

func (pac *Package) createFile(file string) (io.WriteCloser, error) {
	if err := os.MkdirAll(path.Dir(file), 0755); err != nil {
		return nil, err
	}
	f, err := os.Create(file)
	if err != nil {
		return nil, err
	}
	return f, nil
}

func (pac *Package) Wrap() error {
	g, err := pac.createFile(pac.goFile())
	if err != nil {
		return err
	}
	defer g.Close()
	c, err := pac.createFile(pac.cFile())
	if err != nil {
		return err
	}
	defer c.Close()
	h, err := pac.createFile(pac.hFile())
	if err != nil {
		return err
	}
	defer h.Close()
	if err := pac.prepare(); err != nil {
		return err
	}
	if err := pac.write(g, c, h); err != nil {
		return err
	}
	return gofmt(pac.goFile())
}

func (pac *Package) prepare() error {
	if pac.XmlDoc == nil {
		if err := pac.Load(); err != nil {
			return err
		}
	}
	// reset localNames
	pac.localNames = make(map[string]string)
	return nil
}

func (pac *Package) GenConst(file string) error {
	f, err := os.Create(file)
	if err != nil {
		return err
	}
	defer f.Close()
	ms, err := gcc.Xml{pac.From.FullPath()}.Macros()
	if err != nil {
		return err
	}
	consts := ms.Constants(pac.From.NamePattern)
	nm := make(map[string]string)
	for _, c := range consts {
		nm[c.Name] = upperName(c.Name, pac.pat)
	}

	fp(f, "package ", pac.PacName)
	fp(f, "")
	fp(f, "const (")
	for _, c := range consts {
		body := c.Body
		for k, v := range nm {
			body = replace(body, k, v)
		}
		fp(f, upperName(c.Name, pac.pat), "=", body)
	}
	fp(f, ")")
	return nil
}

type Statistics struct {
	DefCount int
}

func (s Statistics) Print() {
	p(s.DefCount, "declarations wrapped.")
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"net"
	"os"
	"path/filepath"
	"strconv"
	"time"

	"github.com/ziutek/dvb"
	"github.com/ziutek/dvb/linuxdvb/demux"
	"github.com/ziutek/dvb/linuxdvb/frontend"
	"github.com/ziutek/dvb/ts"
)

func die(s string) {
	fmt.Fprintln(os.Stderr, s)
	os.Exit(1)
}

func checkErr(err error) {
	if err != nil {
		if err == dvb.ErrOverflow || err == ts.ErrSync {
			return
		}
		die(err.Error())
	}
}

func usage() {
	fmt.Fprintf(
		os.Stderr,
		"Usage: %s [OPTION] PID [PID...]\nOptions:\n",
		filepath.Base(os.Args[0]),
	)
	flag.PrintDefaults()
	os.Exit(1)
}

func main() {
	src := flag.String("src", "rf", "source: rf, udp")
	laddr := flag.String("laddr", "0.0.0.0:1234", "listen on laddr")
	fpath := flag.String("front", "/dev/dvb/adapter0/frontend0", "path to frontend device")
	dpath := flag.String("demux", "/dev/dvb/adapter0/demux0", "path to demux device")
	sys := flag.String("sys", "t", "name of delivery system: t, s, s2, ca, cb, cc")
	freq := flag.Float64("freq", 0, "frequency [Mhz]")
	sr := flag.Uint("sr", 0, "symbol rate [kBd]")
	pol := flag.String("pol", "h", "polarization: h, v")
	count := flag.Uint64("count", 0, "number of MPEG-TS packets to process (0 means infinity)")
	bw := flag.Uint("bw", 0, "bandwidth [MHz] (0 == auto)")
	flag.Usage = usage
	flag.Parse()

	if flag.NArg() == 0 {
		usage()
	}

	pids := make([]int16, flag.NArg())
	for i, a := range flag.Args() {
		pid, err := strconv.ParseInt(a, 0, 64)
		checkErr(err)
		if uint64(pid) > 8192 {
			die(a + " isn't in valid PID range [0, 8192]")
		}
		pids[i] = int16(pid)
	}

	var r ts.PktReader

	switch *src {
	case "rf":
		r = tune(*fpath, *dpath, *sys, *pol, uint64(*freq*1e6), uint64(*bw)*1e6, *sr, pids)
	case "udp":
		r = listenUDP(*laddr, pids)
	default:
		die("Unknown source: " + *src)
	}

	pkt := new(ts.ArrayPkt)

	if *count == 0 {
		for {
			checkErr(r.ReadPkt(pkt))
			_, err := os.Stdout.Write(pkt.Bytes())
			checkErr(err)
		}
		return
	}
	for *count != 0 {
		checkErr(r.ReadPkt(pkt))
		_, err := os.Stdout.Write(pkt.Bytes())
		checkErr(err)
		*count--
	}
}

type pidFilter struct {
	r    ts.PktReader
	pids []int16
}

func (f *pidFilter) ReadPkt(pkt ts.Pkt) error {
	for {
		if err := f.r.ReadPkt(pkt); err != nil {
			return err
		}
		pid := pkt.Pid()
		// TODO: sort f.pids to use more effecitve search method.
		for _, p := range f.pids {
			if p == 8192 || p == pid {
				return nil
			}
		}
	}
}

func listenUDP(laddr string, pids []int16) ts.PktReader {
	la, err := net.ResolveUDPAddr("udp", laddr)
	checkErr(err)
	c, err := net.ListenUDP("udp", la)
	checkErr(err)
	checkErr(c.SetReadBuffer(2 * 1024 * 1024))
	return &pidFilter{
		r:    ts.NewPktPktReader(c, make([]byte, 7*ts.PktLen)),
		pids: pids,
	}
}

func tune(fpath, dpath, sys, pol string, freqHz, bwHz uint64, sr uint, pids []int16) ts.PktReader {
	var polar rune
	switch pol {
	case "h", "v":
		polar = rune((pol)[0])
	default:
		die("unknown polarization: " + pol)
	}

	fe, err := frontend.Open(fpath)
	checkErr(err)

	switch sys {
	case "t":
		checkErr(fe.SetDeliverySystem(dvb.SysDVBT))
		checkErr(fe.SetModulation(dvb.QAMAuto))
		checkErr(fe.SetFrequency(uint32(freqHz)))
		checkErr(fe.SetInversion(dvb.InversionAuto))
		if bwHz != 0 {
			checkErr(fe.SetBandwidth(uint32(bwHz)))
		}
		checkErr(fe.SetCodeRateHP(dvb.FECAuto))
		checkErr(fe.SetCodeRateLP(dvb.FECAuto))
		checkErr(fe.SetTxMode(dvb.TxModeAuto))
		checkErr(fe.SetGuard(dvb.GuardAuto))
		checkErr(fe.SetHierarchy(dvb.HierarchyNone))
	case "s", "s2":
		if sys == "s" {
			checkErr(fe.SetDeliverySystem(dvb.SysDVBS))
			checkErr(fe.SetModulation(dvb.QPSK))
		} else {
			checkErr(fe.SetDeliverySystem(dvb.SysDVBS2))
			checkErr(fe.SetModulation(dvb.PSK8))
			checkErr(fe.SetRolloff(dvb.RolloffAuto))
			checkErr(fe.SetPilot(dvb.PilotAuto))
		}
		checkErr(fe.SetSymbolRate(uint32(sr)))
		checkErr(fe.SetInnerFEC(dvb.FECAuto))
		checkErr(fe.SetInversion(dvb.InversionAuto))
		ifreq, tone, volt := frontend.SecParam(freqHz, polar)
		checkErr(fe.SetFrequency(ifreq))
		checkErr(fe.SetTone(tone))
		checkErr(fe.SetVoltage(volt))
	case "ca", "cb", "cc":
		switch sys {
		case "ca":
			checkErr(fe.SetDeliverySystem(dvb.SysDVBCAnnexA))
		case "cb":
			checkErr(fe.SetDeliverySystem(dvb.SysDVBCAnnexB))
		case "cc":
			checkErr(fe.SetDeliverySystem(dvb.SysDVBCAnnexC))
		}
		checkErr(fe.SetModulation(dvb.QAMAuto))
		checkErr(fe.SetFrequency(uint32(freqHz)))
		checkErr(fe.SetInversion(dvb.InversionAuto))
		checkErr(fe.SetSymbolRate(uint32(sr)))
		checkErr(fe.SetInnerFEC(dvb.FECAuto))
	default:
		die("unknown delivery system: " + sys)
	}

	checkErr(fe.Tune())
	checkErr(waitForTune(fe))

	var filterParam = demux.StreamFilterParam{
		Pid:  pids[0],
		In:   demux.InFrontend,
		Out:  demux.OutTSDemuxTap,
		Type: demux.Other,
	}
	f, err := demux.Device(dpath).StreamFilter(&filterParam)
	checkErr(err)
	for _, pid := range pids[1:] {
		checkErr(f.AddPid(pid))
	}
	checkErr(f.SetBufferLen(1024 * 188))
	checkErr(f.Start())

	return ts.NewPktStreamReader(f)
}

func waitForTune(fe frontend.Device) error {
	fe3 := frontend.API3{fe}
	deadline := time.Now().Add(5 * time.Second)
	var ev frontend.Event
	for ev.Status()&frontend.HasLock == 0 {
		timedout, err := fe3.WaitEvent(&ev, deadline)
		if err != nil {
			return err
		}
		if timedout {
			return errors.New("tuning timeout")
		}
	}
	return nil
}
<commit_msg>Type of freq,bw changed<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"net"
	"os"
	"path/filepath"
	"strconv"
	"time"

	"github.com/ziutek/dvb"
	"github.com/ziutek/dvb/linuxdvb/demux"
	"github.com/ziutek/dvb/linuxdvb/frontend"
	"github.com/ziutek/dvb/ts"
)

func die(s string) {
	fmt.Fprintln(os.Stderr, s)
	os.Exit(1)
}

func checkErr(err error) {
	if err != nil {
		if err == dvb.ErrOverflow || err == ts.ErrSync {
			return
		}
		die(err.Error())
	}
}

func usage() {
	fmt.Fprintf(
		os.Stderr,
		"Usage: %s [OPTION] PID [PID...]\nOptions:\n",
		filepath.Base(os.Args[0]),
	)
	flag.PrintDefaults()
	os.Exit(1)
}

func main() {
	src := flag.String("src", "rf", "source: rf, udp")
	laddr := flag.String("laddr", "0.0.0.0:1234", "listen on laddr")
	fpath := flag.String("front", "/dev/dvb/adapter0/frontend0", "path to frontend device")
	dpath := flag.String("demux", "/dev/dvb/adapter0/demux0", "path to demux device")
	sys := flag.String("sys", "t", "name of delivery system: t, s, s2, ca, cb, cc")
	freq := flag.Float64("freq", 0, "frequency [Mhz]")
	sr := flag.Uint("sr", 0, "symbol rate [kBd]")
	pol := flag.String("pol", "h", "polarization: h, v")
	count := flag.Uint64("count", 0, "number of MPEG-TS packets to process (0 means infinity)")
	bw := flag.Int("bw", 0, "bandwidth [MHz] (0 == auto)")
	flag.Usage = usage
	flag.Parse()

	if flag.NArg() == 0 {
		usage()
	}

	pids := make([]int16, flag.NArg())
	for i, a := range flag.Args() {
		pid, err := strconv.ParseInt(a, 0, 64)
		checkErr(err)
		if uint64(pid) > 8192 {
			die(a + " isn't in valid PID range [0, 8192]")
		}
		pids[i] = int16(pid)
	}

	var r ts.PktReader

	switch *src {
	case "rf":
		r = tune(*fpath, *dpath, *sys, *pol, int64(*freq*1e6), *bw*1e6, *sr, pids)
	case "udp":
		r = listenUDP(*laddr, pids)
	default:
		die("Unknown source: " + *src)
	}

	pkt := new(ts.ArrayPkt)

	if *count == 0 {
		for {
			checkErr(r.ReadPkt(pkt))
			_, err := os.Stdout.Write(pkt.Bytes())
			checkErr(err)
		}
		return
	}
	for *count != 0 {
		checkErr(r.ReadPkt(pkt))
		_, err := os.Stdout.Write(pkt.Bytes())
		checkErr(err)
		*count--
	}
}

type pidFilter struct {
	r    ts.PktReader
	pids []int16
}

func (f *pidFilter) ReadPkt(pkt ts.Pkt) error {
	for {
		if err := f.r.ReadPkt(pkt); err != nil {
			return err
		}
		pid := pkt.Pid()
		// TODO: sort f.pids to use more effecitve search method.
		for _, p := range f.pids {
			if p == 8192 || p == pid {
				return nil
			}
		}
	}
}

func listenUDP(laddr string, pids []int16) ts.PktReader {
	la, err := net.ResolveUDPAddr("udp", laddr)
	checkErr(err)
	c, err := net.ListenUDP("udp", la)
	checkErr(err)
	checkErr(c.SetReadBuffer(2 * 1024 * 1024))
	return &pidFilter{
		r:    ts.NewPktPktReader(c, make([]byte, 7*ts.PktLen)),
		pids: pids,
	}
}

func tune(fpath, dpath, sys, pol string, freqHz int64, bwHz int, sr uint, pids []int16) ts.PktReader {
	var polar rune
	switch pol {
	case "h", "v":
		polar = rune((pol)[0])
	default:
		die("unknown polarization: " + pol)
	}

	fe, err := frontend.Open(fpath)
	checkErr(err)

	switch sys {
	case "t":
		checkErr(fe.SetDeliverySystem(dvb.SysDVBT))
		checkErr(fe.SetModulation(dvb.QAMAuto))
		checkErr(fe.SetFrequency(uint32(freqHz)))
		checkErr(fe.SetInversion(dvb.InversionAuto))
		if bwHz != 0 {
			checkErr(fe.SetBandwidth(uint32(bwHz)))
		}
		checkErr(fe.SetCodeRateHP(dvb.FECAuto))
		checkErr(fe.SetCodeRateLP(dvb.FECAuto))
		checkErr(fe.SetTxMode(dvb.TxModeAuto))
		checkErr(fe.SetGuard(dvb.GuardAuto))
		checkErr(fe.SetHierarchy(dvb.HierarchyNone))
	case "s", "s2":
		if sys == "s" {
			checkErr(fe.SetDeliverySystem(dvb.SysDVBS))
			checkErr(fe.SetModulation(dvb.QPSK))
		} else {
			checkErr(fe.SetDeliverySystem(dvb.SysDVBS2))
			checkErr(fe.SetModulation(dvb.PSK8))
			checkErr(fe.SetRolloff(dvb.RolloffAuto))
			checkErr(fe.SetPilot(dvb.PilotAuto))
		}
		checkErr(fe.SetSymbolRate(uint32(sr)))
		checkErr(fe.SetInnerFEC(dvb.FECAuto))
		checkErr(fe.SetInversion(dvb.InversionAuto))
		ifreq, tone, volt := frontend.SecParam(freqHz, polar)
		checkErr(fe.SetFrequency(ifreq))
		checkErr(fe.SetTone(tone))
		checkErr(fe.SetVoltage(volt))
	case "ca", "cb", "cc":
		switch sys {
		case "ca":
			checkErr(fe.SetDeliverySystem(dvb.SysDVBCAnnexA))
		case "cb":
			checkErr(fe.SetDeliverySystem(dvb.SysDVBCAnnexB))
		case "cc":
			checkErr(fe.SetDeliverySystem(dvb.SysDVBCAnnexC))
		}
		checkErr(fe.SetModulation(dvb.QAMAuto))
		checkErr(fe.SetFrequency(uint32(freqHz)))
		checkErr(fe.SetInversion(dvb.InversionAuto))
		checkErr(fe.SetSymbolRate(uint32(sr)))
		checkErr(fe.SetInnerFEC(dvb.FECAuto))
	default:
		die("unknown delivery system: " + sys)
	}

	checkErr(fe.Tune())
	checkErr(waitForTune(fe))

	var filterParam = demux.StreamFilterParam{
		Pid:  pids[0],
		In:   demux.InFrontend,
		Out:  demux.OutTSDemuxTap,
		Type: demux.Other,
	}
	f, err := demux.Device(dpath).StreamFilter(&filterParam)
	checkErr(err)
	for _, pid := range pids[1:] {
		checkErr(f.AddPid(pid))
	}
	checkErr(f.SetBufferLen(1024 * 188))
	checkErr(f.Start())

	return ts.NewPktStreamReader(f)
}

func waitForTune(fe frontend.Device) error {
	fe3 := frontend.API3{fe}
	deadline := time.Now().Add(5 * time.Second)
	var ev frontend.Event
	for ev.Status()&frontend.HasLock == 0 {
		timedout, err := fe3.WaitEvent(&ev, deadline)
		if err != nil {
			return err
		}
		if timedout {
			return errors.New("tuning timeout")
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package stats

import (
	"io"
	"net"
	"sync"
	"time"

	log "github.com/sirupsen/logrus"
)

var (
	queueItems      *Range32
	genDataDuration *Gauge32
	flushDuration   *LatencyHistogram15s32
	messageSize     *Gauge32
	connected       *Bool
)

type GraphiteMetric interface {
	// WriteGraphiteLine appends the Graphite formatted metric measurement to `buf` and resets measurements for the next interval if needed
	// `buf` is the incoming buffer to be appended to
	// `prefix` is an optional prefix to the metric name which must have a trailing '.' if present
	// `now` is the time that the metrics should be reported at
	WriteGraphiteLine(buf, prefix []byte, now time.Time) []byte
}

type Graphite struct {
	prefix []byte
	addr   string

	timeout    time.Duration
	toGraphite chan []byte
}

func NewGraphite(prefix, addr string, interval, bufferSize int, timeout time.Duration) {
	if len(prefix) != 0 && prefix[len(prefix)-1] != '.' {
		prefix = prefix + "."
	}
	NewGauge32("stats.graphite.write_queue.size").Set(bufferSize)
	queueItems = NewRange32("stats.graphite.write_queue.items")
	// metric stats.generate_message is how long it takes to generate the stats
	genDataDuration = NewGauge32("stats.generate_message.duration")
	flushDuration = NewLatencyHistogram15s32("stats.graphite.flush")
	messageSize = NewGauge32("stats.message_size")
	connected = NewBool("stats.graphite.connected")

	g := &Graphite{
		prefix:     []byte(prefix),
		addr:       addr,
		toGraphite: make(chan []byte, bufferSize),
		timeout:    timeout,
	}
	go g.writer()
	go g.reporter(interval)
}

func (g *Graphite) reporter(interval int) {
	ticker := tick(time.Duration(interval) * time.Second)
	for now := range ticker {
		log.Debugf("stats flushing for %s to graphite", now)
		queueItems.Value(len(g.toGraphite))
		if cap(g.toGraphite) != 0 && len(g.toGraphite) == cap(g.toGraphite) {
			// no space in buffer, no use in doing any work
			continue
		}

		pre := time.Now()

		buf := make([]byte, 0)

		for _, metric := range registry.list() {
			buf = metric.WriteGraphiteLine(buf, g.prefix, now)
		}

		genDataDuration.Set(int(time.Since(pre).Nanoseconds()))
		messageSize.Set(len(buf))
		g.toGraphite <- buf
		queueItems.Value(len(g.toGraphite))
	}
}

// writer connects to graphite and submits all pending data to it
func (g *Graphite) writer() {
	var conn net.Conn
	var err error
	var wg sync.WaitGroup

	assureConn := func() {
		connected.Set(conn != nil)
		for conn == nil {
			time.Sleep(time.Second)
			conn, err = net.Dial("tcp", g.addr)
			if err == nil {
				log.Infof("stats now connected to %s", g.addr)
				wg.Add(1)
				go g.checkEOF(conn, &wg)
			} else {
				log.Warnf("stats dialing %s failed: %s. will retry", g.addr, err.Error())
			}
			connected.Set(conn != nil)
		}
	}

	for buf := range g.toGraphite {
		queueItems.Value(len(g.toGraphite))
		var ok bool
		for !ok {
			assureConn()
			conn.SetWriteDeadline(time.Now().Add(g.timeout))
			pre := time.Now()
			_, err = conn.Write(buf)
			if err == nil {
				ok = true
				flushDuration.Value(time.Since(pre))
			} else {
				log.Warnf("stats failed to write to graphite: %s (took %s). will retry...", err, time.Now().Sub(pre))
				conn.Close()
				wg.Wait()
				conn = nil
			}
		}
	}
}

// normally the remote end should never write anything back
// but we know when we get EOF that the other end closed the conn
// if not for this, we can happily write and flush without getting errors (in Go) but getting RST tcp packets back (!)
// props to Tv` for this trick.
func (g *Graphite) checkEOF(conn net.Conn, wg *sync.WaitGroup) {
	defer wg.Done()
	b := make([]byte, 1024)
	for {
		num, err := conn.Read(b)
		if err == io.EOF {
			log.Info("Graphite.checkEOF: remote closed conn. closing conn")
			conn.Close()
			return
		}

		// in case the remote behaves badly (out of spec for carbon protocol)
		if num != 0 {
			log.Warnf("Graphite.checkEOF: read unexpected data from peer: %s\n", b[:num])
			continue
		}

		if err != io.EOF {
			log.Warnf("Graphite.checkEOF: %s. closing conn\n", err)
			conn.Close()
			return
		}
	}
}
<commit_msg>Update Graphite struct to allow manual reporting for given timestamp.<commit_after>package stats

import (
	"io"
	"net"
	"sync"
	"time"

	log "github.com/sirupsen/logrus"
)

var (
	queueItems      *Range32
	genDataDuration *Gauge32
	flushDuration   *LatencyHistogram15s32
	messageSize     *Gauge32
	connected       *Bool
)

type GraphiteMetric interface {
	// WriteGraphiteLine appends the Graphite formatted metric measurement to `buf` and resets measurements for the next interval if needed
	// `buf` is the incoming buffer to be appended to
	// `prefix` is an optional prefix to the metric name which must have a trailing '.' if present
	// `now` is the time that the metrics should be reported at
	WriteGraphiteLine(buf, prefix []byte, now time.Time) []byte
}

type Graphite struct {
	prefix []byte
	addr   string

	timeout    time.Duration
	toGraphite chan []byte
}

//NewGraphite creates and starts a graphite reporter which.
//prefix is a string prefix which is added to every metric
//addr is the graphite address to report to
//interval is the interval in seconds that metrics should be reported.  If interval is negative, metrics will not be reported automatically.
//bufferSize determines how many reporting intervals should be buffered in memory.  If full, new intervals will not be reported
//timeout determines how long to wait while reporting an interval
//returns a new graphite instance which should only be used for manual reporting if interval is < 0
func NewGraphite(prefix, addr string, interval, bufferSize int, timeout time.Duration) *Graphite {
	if len(prefix) != 0 && prefix[len(prefix)-1] != '.' {
		prefix = prefix + "."
	}
	NewGauge32("stats.graphite.write_queue.size").Set(bufferSize)
	queueItems = NewRange32("stats.graphite.write_queue.items")
	// metric stats.generate_message is how long it takes to generate the stats
	genDataDuration = NewGauge32("stats.generate_message.duration")
	flushDuration = NewLatencyHistogram15s32("stats.graphite.flush")
	messageSize = NewGauge32("stats.message_size")
	connected = NewBool("stats.graphite.connected")

	g := &Graphite{
		prefix:     []byte(prefix),
		addr:       addr,
		toGraphite: make(chan []byte, bufferSize),
		timeout:    timeout,
	}
	go g.writer()
	if interval > 0 {
		go g.reporter(interval)
	}
	return g
}

//Report sends graphite metrics with the given timestamp.
//This should only be used if a negative reporting interval has been set.
func (g *Graphite) Report(now time.Time) {
	log.Debugf("stats flushing for %s to graphite", now)
	queueItems.Value(len(g.toGraphite))
	if cap(g.toGraphite) != 0 && len(g.toGraphite) == cap(g.toGraphite) {
		// no space in buffer, no use in doing any work
		return
	}

	pre := time.Now()

	buf := make([]byte, 0)

	for _, metric := range registry.list() {
		buf = metric.WriteGraphiteLine(buf, g.prefix, now)
	}

	genDataDuration.Set(int(time.Since(pre).Nanoseconds()))
	messageSize.Set(len(buf))
	g.toGraphite <- buf
	queueItems.Value(len(g.toGraphite))
}

func (g *Graphite) reporter(interval int) {
	ticker := tick(time.Duration(interval) * time.Second)
	for now := range ticker {
		g.Report(now)
	}
}

// writer connects to graphite and submits all pending data to it
func (g *Graphite) writer() {
	var conn net.Conn
	var err error
	var wg sync.WaitGroup

	assureConn := func() {
		connected.Set(conn != nil)
		for conn == nil {
			time.Sleep(time.Second)
			conn, err = net.Dial("tcp", g.addr)
			if err == nil {
				log.Infof("stats now connected to %s", g.addr)
				wg.Add(1)
				go g.checkEOF(conn, &wg)
			} else {
				log.Warnf("stats dialing %s failed: %s. will retry", g.addr, err.Error())
			}
			connected.Set(conn != nil)
		}
	}

	for buf := range g.toGraphite {
		queueItems.Value(len(g.toGraphite))
		var ok bool
		for !ok {
			assureConn()
			conn.SetWriteDeadline(time.Now().Add(g.timeout))
			pre := time.Now()
			_, err = conn.Write(buf)
			if err == nil {
				ok = true
				flushDuration.Value(time.Since(pre))
			} else {
				log.Warnf("stats failed to write to graphite: %s (took %s). will retry...", err, time.Now().Sub(pre))
				conn.Close()
				wg.Wait()
				conn = nil
			}
		}
	}
}

// normally the remote end should never write anything back
// but we know when we get EOF that the other end closed the conn
// if not for this, we can happily write and flush without getting errors (in Go) but getting RST tcp packets back (!)
// props to Tv` for this trick.
func (g *Graphite) checkEOF(conn net.Conn, wg *sync.WaitGroup) {
	defer wg.Done()
	b := make([]byte, 1024)
	for {
		num, err := conn.Read(b)
		if err == io.EOF {
			log.Info("Graphite.checkEOF: remote closed conn. closing conn")
			conn.Close()
			return
		}

		// in case the remote behaves badly (out of spec for carbon protocol)
		if num != 0 {
			log.Warnf("Graphite.checkEOF: read unexpected data from peer: %s\n", b[:num])
			continue
		}

		if err != io.EOF {
			log.Warnf("Graphite.checkEOF: %s. closing conn\n", err)
			conn.Close()
			return
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"log"
	"runtime"
	"time"

	"github.com/rakyll/gcptrace"
)

var t *trace.Client

func main() {
	ctx, err := trace.New(context.Background(), "jbd-gce")
	if err != nil {
		log.Fatal(err)
	}
	f(ctx)
}

func f(ctx context.Context) {
	ctx = trace.WithSpan(ctx, "")
	defer trace.Finish(ctx)

	go a1(ctx)
	a2(ctx)
	a3(ctx)
}

func a1(ctx context.Context) {
	ctx = trace.WithSpan(ctx, "")
	defer trace.Finish(ctx)

	trace.Logf(ctx, "this is a format string, num goroutines: %v", runtime.NumGoroutine())
	time.Sleep(100 * time.Millisecond)
}

func a2(ctx context.Context) {
	ctx = trace.WithSpan(ctx, "a2")
	defer trace.Finish(ctx)

	time.Sleep(200 * time.Millisecond)
}

func a3(ctx context.Context) {
	ctx = trace.WithSpan(ctx, "a3")
	defer trace.Finish(ctx)

	time.Sleep(300 * time.Millisecond)
}
<commit_msg>remove unused trace.Client<commit_after>package main

import (
	"context"
	"log"
	"runtime"
	"time"

	"github.com/rakyll/gcptrace"
)

func main() {
	ctx, err := trace.New(context.Background(), "jbd-gce")
	if err != nil {
		log.Fatal(err)
	}
	f(ctx)
}

func f(ctx context.Context) {
	ctx = trace.WithSpan(ctx, "")
	defer trace.Finish(ctx)

	go a1(ctx)
	a2(ctx)
	a3(ctx)
}

func a1(ctx context.Context) {
	ctx = trace.WithSpan(ctx, "")
	defer trace.Finish(ctx)

	trace.Logf(ctx, "this is a format string, num goroutines: %v", runtime.NumGoroutine())
	time.Sleep(100 * time.Millisecond)
}

func a2(ctx context.Context) {
	ctx = trace.WithSpan(ctx, "a2")
	defer trace.Finish(ctx)

	time.Sleep(200 * time.Millisecond)
}

func a3(ctx context.Context) {
	ctx = trace.WithSpan(ctx, "a3")
	defer trace.Finish(ctx)

	time.Sleep(300 * time.Millisecond)
}
<|endoftext|>


<commit_before>package storage

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
)

type Filesystem struct {
	Root string
	c    uint64
	mu   sync.RWMutex
}

func NewFilesystem(root string) (*Filesystem, error) {
	s := &Filesystem{
		Root: root,
	}
	return s, os.MkdirAll(s.Root, 0744)
}

func (s *Filesystem) Code(url string) string {
	return strconv.FormatUint(s.c, 36)
}

func (s *Filesystem) Save(url string) (string, error) {
	if url == "" {
		return "", ErrURLEmpty
	}

	code := s.Code(url)

	s.mu.Lock()
	err := ioutil.WriteFile(filepath.Join(s.Root, code), []byte(url), 0744)
	if err == nil {
		s.c++
	}
	s.mu.Unlock()

	return code, err
}

// CleanPath removes any path transversal nonsense
func CleanPath(path string) string {
	if !strings.HasPrefix(path, "/") {
		path = "/" + path
	}

	return filepath.Clean(path)
}

// Takes a possibly multilevel path and flattens it by dropping any slashes
func FlattenPath(path string, separator string) {
	return strings.Replace(path, os.PathSeparator, separator, -1)
}

func (s *Filesystem) SaveName(short, long string) error {
	if short == "" {
		return ErrNameEmpty
	}
	if long == "" {
		return ErrURLEmpty
	}

	short = FlattenPath(CleanPath(short), "_")

	s.mu.Lock()
	err := ioutil.WriteFile(filepath.Join(s.Root, short), []byte(long), 0744)
	if err == nil {
		s.c++
	}
	s.mu.Unlock()

	return err
}

func (s *Filesystem) Load(code string) (string, error) {
	if code == "" {
		return "", ErrNameEmpty
	}

	code = FlattenPath(CleanPath(code), "_")

	s.mu.Lock()
	urlBytes, err := ioutil.ReadFile(filepath.Join(s.Root, code))
	s.mu.Unlock()

	if _, ok := err.(*os.PathError); ok {
		return "", ErrCodeNotSet
	}

	return string(urlBytes), err
}
<commit_msg>Fix bug in FlattenPath<commit_after>package storage

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
)

type Filesystem struct {
	Root string
	c    uint64
	mu   sync.RWMutex
}

func NewFilesystem(root string) (*Filesystem, error) {
	s := &Filesystem{
		Root: root,
	}
	return s, os.MkdirAll(s.Root, 0744)
}

func (s *Filesystem) Code(url string) string {
	return strconv.FormatUint(s.c, 36)
}

func (s *Filesystem) Save(url string) (string, error) {
	if url == "" {
		return "", ErrURLEmpty
	}

	code := s.Code(url)

	s.mu.Lock()
	err := ioutil.WriteFile(filepath.Join(s.Root, code), []byte(url), 0744)
	if err == nil {
		s.c++
	}
	s.mu.Unlock()

	return code, err
}

// CleanPath removes any path transversal nonsense
func CleanPath(path string) string {
	if !strings.HasPrefix(path, "/") {
		path = "/" + path
	}

	return filepath.Clean(path)
}

// Takes a possibly multilevel path and flattens it by dropping any slashes
func FlattenPath(path string, separator string) string {
	return strings.Replace(path, string(os.PathSeparator), separator, -1)
}

func (s *Filesystem) SaveName(short, long string) error {
	if short == "" {
		return ErrNameEmpty
	}
	if long == "" {
		return ErrURLEmpty
	}

	short = FlattenPath(CleanPath(short), "_")

	s.mu.Lock()
	err := ioutil.WriteFile(filepath.Join(s.Root, short), []byte(long), 0744)
	if err == nil {
		s.c++
	}
	s.mu.Unlock()

	return err
}

func (s *Filesystem) Load(code string) (string, error) {
	if code == "" {
		return "", ErrNameEmpty
	}

	code = FlattenPath(CleanPath(code), "_")

	s.mu.Lock()
	urlBytes, err := ioutil.ReadFile(filepath.Join(s.Root, code))
	s.mu.Unlock()

	if _, ok := err.(*os.PathError); ok {
		return "", ErrCodeNotSet
	}

	return string(urlBytes), err
}
<|endoftext|>


<commit_before>/*

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"os"

	"contrib.go.opencensus.io/exporter/prometheus"
	"contrib.go.opencensus.io/exporter/stackdriver"
	prom "github.com/prometheus/client_golang/prometheus"
	"go.opencensus.io/stats/view"
	corev1 "k8s.io/api/core/v1"

	// Change to use v1 when we only need to support 1.17 and higher kubernetes versions.
	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
	"k8s.io/apimachinery/pkg/runtime"
	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/log/zap"

	// +kubebuilder:scaffold:imports

	v1a1 "sigs.k8s.io/multi-tenancy/incubator/hnc/api/v1alpha1"
	v1a2 "sigs.k8s.io/multi-tenancy/incubator/hnc/api/v1alpha2"
	"sigs.k8s.io/multi-tenancy/incubator/hnc/internal/forest"
	"sigs.k8s.io/multi-tenancy/incubator/hnc/internal/reconcilers"
	"sigs.k8s.io/multi-tenancy/incubator/hnc/internal/stats"
	"sigs.k8s.io/multi-tenancy/incubator/hnc/internal/validators"
)

var (
	scheme   = runtime.NewScheme()
	setupLog = zap.New().WithName("setup")
)

var (
	metricsAddr          string
	maxReconciles        int
	enableLeaderElection bool
	leaderElectionId     string
	novalidation         bool
	debugLogs            bool
	testLog              bool
	internalCert         bool
	qps                  int
	webhookServerPort    int
)

func init() {
	setupLog.Info("Starting main.go:init()")
	defer setupLog.Info("Finished main.go:init()")
	_ = clientgoscheme.AddToScheme(scheme)

	_ = v1a1.AddToScheme(scheme)
	_ = v1a2.AddToScheme(scheme)
	_ = corev1.AddToScheme(scheme)
	_ = v1beta1.AddToScheme(scheme)
	// +kubebuilder:scaffold:scheme
}

func main() {
	setupLog.Info("Parsing flags")
	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
		"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
	flag.StringVar(&leaderElectionId, "leader-election-id", "controller-leader-election-helper",
		"Leader election id determines the name of the configmap that leader election will use for holding the leader lock.")
	flag.BoolVar(&novalidation, "novalidation", false, "Disables validating webhook")
	flag.BoolVar(&debugLogs, "debug-logs", false, "Shows verbose logs in a human-friendly format.")
	flag.BoolVar(&testLog, "enable-test-log", false, "Enables test log.")
	flag.BoolVar(&internalCert, "enable-internal-cert-management", false, "Enables internal cert management.")
	flag.IntVar(&maxReconciles, "max-reconciles", 1, "Number of concurrent reconciles to perform.")
	flag.IntVar(&qps, "apiserver-qps-throttle", 50, "The maximum QPS to the API server.")
	flag.BoolVar(&stats.SuppressObjectTags, "suppress-object-tags", true, "If true, suppresses the kinds of object metrics to reduce metric cardinality.")
	flag.IntVar(&webhookServerPort, "webhook-server-port", 443, "The port that the webhook server serves at.")
	flag.Parse()

	// Enable OpenCensus exporters to export metrics
	// to Stackdriver Monitoring.
	// Exporters use Application Default Credentials to authenticate.
	// See https://developers.google.com/identity/protocols/application-default-credentials
	// for more details.
	setupLog.Info("Creating OpenCensus->Stackdriver exporter")
	sd, err := stackdriver.NewExporter(stackdriver.Options{
		// Stackdriver’s minimum stats reporting period must be >= 60 seconds.
		// https://opencensus.io/exporters/supported-exporters/go/stackdriver/
		ReportingInterval: stats.ReportingInterval,
	})
	if err == nil {
		// Flush must be called before main() exits to ensure metrics are recorded.
		defer sd.Flush()
		err = sd.StartMetricsExporter()
		if err == nil {
			defer sd.StopMetricsExporter()
		}
	}
	if err != nil {
		setupLog.Error(err, "cannot create Stackdriver exporter")
	}

	setupLog.Info("Creating Prometheus exporter")
	prom.DefaultRegisterer = prom.DefaultRegisterer.(*prom.Registry)
	promExporter, err := prometheus.NewExporter(prometheus.Options{Registry: prom.DefaultRegisterer.(*prom.Registry)})
	if err != nil {
		setupLog.Error(err, "Cannot create Prometheus exporter")
	}
	view.RegisterExporter(promExporter)

	setupLog.Info("Configuring controller-manager")
	ctrl.SetLogger(zap.Logger(debugLogs))
	cfg := ctrl.GetConfigOrDie()
	cfg.QPS = float32(qps)
	// By default, Burst is about 2x QPS, but since HNC's "bursts" can last for ~minutes
	// we need to raise the QPS param to be much higher than we ordinarily would. As a
	// result, doubling this higher threshold is probably much too high, so lower it to a more
	// reasonable number.
	//
	// TODO: Better understand the behaviour of Burst, and consider making it equal to QPS if
	// it turns out to be harmful.
	cfg.Burst = int(cfg.QPS * 1.5)
	mgr, err := ctrl.NewManager(cfg, ctrl.Options{
		Scheme:             scheme,
		MetricsBindAddress: metricsAddr,
		LeaderElection:     enableLeaderElection,
		LeaderElectionID:   leaderElectionId,
		Port:               webhookServerPort,
	})
	if err != nil {
		setupLog.Error(err, "unable to start manager")
		os.Exit(1)
	}

	// Make sure certs are generated and valid if webhooks are enabled and internal certs are used.
	setupLog.Info("Starting certificate generation")
	setupFinished, err := validators.CreateCertsIfNeeded(mgr, novalidation, internalCert)
	if err != nil {
		setupLog.Error(err, "unable to set up cert rotation")
		os.Exit(1)
	}

	go startControllers(mgr, setupFinished)

	setupLog.Info("Starting manager")
	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
		setupLog.Error(err, "problem running manager")
		os.Exit(1)
	}
}

func startControllers(mgr ctrl.Manager, setupFinished chan struct{}) {
	setupLog.Info("Waiting for certificate generation to complete")
	// Block until the setup finishes.
	<-setupFinished

	if testLog {
		stats.StartLoggingActivity()
	}

	// Create all reconciling controllers
	f := forest.NewForest()
	setupLog.Info("Creating controllers", "maxReconciles", maxReconciles)
	removeOldCRDVersion := true
	if err := reconcilers.Create(mgr, f, maxReconciles, removeOldCRDVersion); err != nil {
		setupLog.Error(err, "cannot create controllers")
		os.Exit(1)
	}

	// Create all validating admission controllers.
	if !novalidation {
		setupLog.Info("Registering validating webhook (won't work when running locally; use --novalidation)")
		validators.Create(mgr, f)
	}

	// Create CRD conversion webhooks.
	if err := (&v1a2.HNCConfiguration{}).SetupWebhookWithManager(mgr); err != nil {
		setupLog.Error(err, "unable to create CRD convension webhook", v1a2.HNCConfigSingletons)
		os.Exit(1)
	}

	if err := (&v1a2.HierarchyConfiguration{}).SetupWebhookWithManager(mgr); err != nil {
		setupLog.Error(err, "unable to create CRD convension webhook", v1a2.HierarchyConfigurations)
		os.Exit(1)
	}

	if err := (&v1a2.SubnamespaceAnchor{}).SetupWebhookWithManager(mgr); err != nil {
		setupLog.Error(err, "unable to create CRD convension webhook", v1a2.Anchors)
		os.Exit(1)
	}

	setupLog.Info("All controllers started; setup complete")
}
<commit_msg>Move webhook setup before manager start<commit_after>/*

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"os"

	"contrib.go.opencensus.io/exporter/prometheus"
	"contrib.go.opencensus.io/exporter/stackdriver"
	prom "github.com/prometheus/client_golang/prometheus"
	"go.opencensus.io/stats/view"
	corev1 "k8s.io/api/core/v1"

	// Change to use v1 when we only need to support 1.17 and higher kubernetes versions.
	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
	"k8s.io/apimachinery/pkg/runtime"
	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/log/zap"

	// +kubebuilder:scaffold:imports

	v1a1 "sigs.k8s.io/multi-tenancy/incubator/hnc/api/v1alpha1"
	v1a2 "sigs.k8s.io/multi-tenancy/incubator/hnc/api/v1alpha2"
	"sigs.k8s.io/multi-tenancy/incubator/hnc/internal/forest"
	"sigs.k8s.io/multi-tenancy/incubator/hnc/internal/reconcilers"
	"sigs.k8s.io/multi-tenancy/incubator/hnc/internal/stats"
	"sigs.k8s.io/multi-tenancy/incubator/hnc/internal/validators"
)

var (
	scheme   = runtime.NewScheme()
	setupLog = zap.New().WithName("setup")
)

var (
	metricsAddr          string
	maxReconciles        int
	enableLeaderElection bool
	leaderElectionId     string
	novalidation         bool
	debugLogs            bool
	testLog              bool
	internalCert         bool
	qps                  int
	webhookServerPort    int
)

func init() {
	setupLog.Info("Starting main.go:init()")
	defer setupLog.Info("Finished main.go:init()")
	_ = clientgoscheme.AddToScheme(scheme)

	_ = v1a1.AddToScheme(scheme)
	_ = v1a2.AddToScheme(scheme)
	_ = corev1.AddToScheme(scheme)
	_ = v1beta1.AddToScheme(scheme)
	// +kubebuilder:scaffold:scheme
}

func main() {
	setupLog.Info("Parsing flags")
	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
		"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
	flag.StringVar(&leaderElectionId, "leader-election-id", "controller-leader-election-helper",
		"Leader election id determines the name of the configmap that leader election will use for holding the leader lock.")
	flag.BoolVar(&novalidation, "novalidation", false, "Disables validating webhook")
	flag.BoolVar(&debugLogs, "debug-logs", false, "Shows verbose logs in a human-friendly format.")
	flag.BoolVar(&testLog, "enable-test-log", false, "Enables test log.")
	flag.BoolVar(&internalCert, "enable-internal-cert-management", false, "Enables internal cert management.")
	flag.IntVar(&maxReconciles, "max-reconciles", 1, "Number of concurrent reconciles to perform.")
	flag.IntVar(&qps, "apiserver-qps-throttle", 50, "The maximum QPS to the API server.")
	flag.BoolVar(&stats.SuppressObjectTags, "suppress-object-tags", true, "If true, suppresses the kinds of object metrics to reduce metric cardinality.")
	flag.IntVar(&webhookServerPort, "webhook-server-port", 443, "The port that the webhook server serves at.")
	flag.Parse()

	// Enable OpenCensus exporters to export metrics
	// to Stackdriver Monitoring.
	// Exporters use Application Default Credentials to authenticate.
	// See https://developers.google.com/identity/protocols/application-default-credentials
	// for more details.
	setupLog.Info("Creating OpenCensus->Stackdriver exporter")
	sd, err := stackdriver.NewExporter(stackdriver.Options{
		// Stackdriver’s minimum stats reporting period must be >= 60 seconds.
		// https://opencensus.io/exporters/supported-exporters/go/stackdriver/
		ReportingInterval: stats.ReportingInterval,
	})
	if err == nil {
		// Flush must be called before main() exits to ensure metrics are recorded.
		defer sd.Flush()
		err = sd.StartMetricsExporter()
		if err == nil {
			defer sd.StopMetricsExporter()
		}
	}
	if err != nil {
		setupLog.Error(err, "cannot create Stackdriver exporter")
	}

	setupLog.Info("Creating Prometheus exporter")
	prom.DefaultRegisterer = prom.DefaultRegisterer.(*prom.Registry)
	promExporter, err := prometheus.NewExporter(prometheus.Options{Registry: prom.DefaultRegisterer.(*prom.Registry)})
	if err != nil {
		setupLog.Error(err, "Cannot create Prometheus exporter")
	}
	view.RegisterExporter(promExporter)

	setupLog.Info("Configuring controller-manager")
	ctrl.SetLogger(zap.Logger(debugLogs))
	cfg := ctrl.GetConfigOrDie()
	cfg.QPS = float32(qps)
	// By default, Burst is about 2x QPS, but since HNC's "bursts" can last for ~minutes
	// we need to raise the QPS param to be much higher than we ordinarily would. As a
	// result, doubling this higher threshold is probably much too high, so lower it to a more
	// reasonable number.
	//
	// TODO: Better understand the behaviour of Burst, and consider making it equal to QPS if
	// it turns out to be harmful.
	cfg.Burst = int(cfg.QPS * 1.5)
	mgr, err := ctrl.NewManager(cfg, ctrl.Options{
		Scheme:             scheme,
		MetricsBindAddress: metricsAddr,
		LeaderElection:     enableLeaderElection,
		LeaderElectionID:   leaderElectionId,
		Port:               webhookServerPort,
	})
	if err != nil {
		setupLog.Error(err, "unable to start manager")
		os.Exit(1)
	}

	// Make sure certs are generated and valid if webhooks are enabled and internal certs are used.
	setupLog.Info("Starting certificate generation")
	setupFinished, err := validators.CreateCertsIfNeeded(mgr, novalidation, internalCert)
	if err != nil {
		setupLog.Error(err, "unable to set up cert rotation")
		os.Exit(1)
	}

	// Register webhooks before manager start to avoid potential race conditions.
	// See https://github.com/kubernetes-sigs/controller-runtime/issues/1148.
	f := forest.NewForest()

	// Create all validating admission controllers.
	if !novalidation {
		setupLog.Info("Registering validating webhook (won't work when running locally; use --novalidation)")
		validators.Create(mgr, f)
	}

	// Create CRD conversion webhooks.
	if err := (&v1a2.HNCConfiguration{}).SetupWebhookWithManager(mgr); err != nil {
		setupLog.Error(err, "unable to create CRD convension webhook", v1a2.HNCConfigSingletons)
		os.Exit(1)
	}

	if err := (&v1a2.HierarchyConfiguration{}).SetupWebhookWithManager(mgr); err != nil {
		setupLog.Error(err, "unable to create CRD convension webhook", v1a2.HierarchyConfigurations)
		os.Exit(1)
	}

	if err := (&v1a2.SubnamespaceAnchor{}).SetupWebhookWithManager(mgr); err != nil {
		setupLog.Error(err, "unable to create CRD convension webhook", v1a2.Anchors)
		os.Exit(1)
	}

	go startControllers(mgr, f, setupFinished)

	setupLog.Info("Starting manager")
	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
		setupLog.Error(err, "problem running manager")
		os.Exit(1)
	}
}

func startControllers(mgr ctrl.Manager, f *forest.Forest, setupFinished chan struct{}) {
	setupLog.Info("Waiting for certificate generation to complete")
	// Block until the setup finishes.
	<-setupFinished

	if testLog {
		stats.StartLoggingActivity()
	}

	// Create all reconciling controllers
	setupLog.Info("Creating controllers", "maxReconciles", maxReconciles)
	removeOldCRDVersion := true
	if err := reconcilers.Create(mgr, f, maxReconciles, removeOldCRDVersion); err != nil {
		setupLog.Error(err, "cannot create controllers")
		os.Exit(1)
	}

	setupLog.Info("All controllers started; setup complete")
}
<|endoftext|>


<commit_before>package webshell

import (
	"log"
	"net/http"
	"path/filepath"
)

// Simple name for a function capable of handling a route.
type RouteHandler func(w http.ResponseWriter, r *http.Request)

// Simple name for a function that handles basic errors.
type ErrorRoute func(string, string, http.ResponseWriter, *http.Request)
type TemplateErrorRoute func(interface{}, http.ResponseWriter, *http.Request)

// Generic error handlers. They take a message and content-type as a string,
// as well as the HTTP response writer and request, and respond with the
// named error. The http.StatusText function may be used to return the
// text for the error code. Note that you have to explicitly call these; the
// design of Go's http server means it will respond with its own 404 handler
// if a route is not found or if a static file server cannot find the file.
var (
	Error400 func(msg, ctype string, w http.ResponseWriter, r *http.Request)
	Error401 func(msg, ctype string, w http.ResponseWriter, r *http.Request)
	Error403 func(msg, ctype string, w http.ResponseWriter, r *http.Request)
	Error404 func(msg, ctype string, w http.ResponseWriter, r *http.Request)
	Error405 func(msg, ctype string, w http.ResponseWriter, r *http.Request)
	Error429 func(msg, ctype string, w http.ResponseWriter, r *http.Request)
	Error500 func(msg, ctype string, w http.ResponseWriter, r *http.Request)
	Error501 func(msg, ctype string, w http.ResponseWriter, r *http.Request)
	Error502 func(msg, ctype string, w http.ResponseWriter, r *http.Request)
	Error503 func(msg, ctype string, w http.ResponseWriter, r *http.Request)
)

// AddRoute adds a route to the app. The route handler needs to match
// the net/http specifications.
func (app *WebApp) AddRoute(path string, handler RouteHandler) {
	app.mux.HandleFunc(path, handler)
	log.Printf("[+] route %s added\n", path)
}

// AddConditionalRoute adds the route and handler if the condition is
// true.
func (app *WebApp) AddConditionalRoute(condition bool, path string, handler RouteHandler) {
	if condition {
		app.AddRoute(path, handler)
	}
}

// StaticRoute adds a new file server route for static assets.
func (app *WebApp) StaticRoute(route string, path string) {
	var err error
	if len(route) == 0 {
		panic("Invalid route: " + route + " -> " + path)
	}
	if len(path) == 0 {
		panic("Invalid path:" + route + " -> " + path)
	} else {
		path, err = filepath.Abs(path)
		if err != nil {
			panic(err)
		}
	}
	app.mux.Handle(route, http.StripPrefix(route, http.FileServer(http.Dir(path))))
	log.Printf("static route %s -> %s added\n", route, path)
}

// GenerateErrorHandler returns a RouteHandler function
func GenerateErrorHandler(status int) ErrorRoute {
	return func(msg, ctype string, w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(status)
		w.Header().Add("content-type", ctype)
		w.Write([]byte(msg))
	}
}

// GenerateTemplateErrorHandler returns a function serving a templated error
func GenerateTemplateErrorHandler(status int, filename string) (hdlr TemplateErrorRoute, err error) {
	tpl, err := CompileTemplate(filename)
	if err != nil {
		return
	}
	hdlr = func(in interface{}, w http.ResponseWriter, r *http.Request) {
		msg, err := BuildTemplate(tpl, in)
		if err != nil {
			log.Printf("error serving template %d %s: %s\n",
				status, filename, err.Error())
			return
		}
		w.WriteHeader(status)
		w.Write(msg)
	}
	return
}

func init() {
	Error400 = GenerateErrorHandler(http.StatusBadRequest)
	Error401 = GenerateErrorHandler(http.StatusUnauthorized)
	Error403 = GenerateErrorHandler(http.StatusForbidden)
	Error404 = GenerateErrorHandler(http.StatusNotFound)
	Error429 = GenerateErrorHandler(429)
	Error500 = GenerateErrorHandler(http.StatusInternalServerError)
	Error501 = GenerateErrorHandler(http.StatusNotImplemented)
	Error502 = GenerateErrorHandler(http.StatusBadGateway)
	Error503 = GenerateErrorHandler(http.StatusServiceUnavailable)

}

// ContentResponder returns the appropriate content-type for the
// request.
func ContentResponder(r *http.Request) string {
	accept := r.Header["Accept"][0]
	if accept == "" {
		return "text/plain"
	} else if accept == "*/*" {
		return "text/plain"
	}
	return accept
}
<commit_msg>rename error route<commit_after>package webshell

import (
	"log"
	"net/http"
	"path/filepath"
)

// Simple name for a function capable of handling a route.
type RouteHandler func(w http.ResponseWriter, r *http.Request)

// Simple name for a function that handles basic errors.
type ErrorHandlerFn func(string, string, http.ResponseWriter, *http.Request)
type TemplateErrorHandlerFn func(interface{}, http.ResponseWriter, *http.Request)

// Generic error handlers. They take a message and content-type as a string,
// as well as the HTTP response writer and request, and respond with the
// named error. The http.StatusText function may be used to return the
// text for the error code. Note that you have to explicitly call these; the
// design of Go's http server means it will respond with its own 404 handler
// if a route is not found or if a static file server cannot find the file.
var (
	Error400 func(msg, ctype string, w http.ResponseWriter, r *http.Request)
	Error401 func(msg, ctype string, w http.ResponseWriter, r *http.Request)
	Error403 func(msg, ctype string, w http.ResponseWriter, r *http.Request)
	Error404 func(msg, ctype string, w http.ResponseWriter, r *http.Request)
	Error405 func(msg, ctype string, w http.ResponseWriter, r *http.Request)
	Error429 func(msg, ctype string, w http.ResponseWriter, r *http.Request)
	Error500 func(msg, ctype string, w http.ResponseWriter, r *http.Request)
	Error501 func(msg, ctype string, w http.ResponseWriter, r *http.Request)
	Error502 func(msg, ctype string, w http.ResponseWriter, r *http.Request)
	Error503 func(msg, ctype string, w http.ResponseWriter, r *http.Request)
)

// AddRoute adds a route to the app. The route handler needs to match
// the net/http specifications.
func (app *WebApp) AddRoute(path string, handler RouteHandler) {
	app.mux.HandleFunc(path, handler)
	log.Printf("[+] route %s added\n", path)
}

// AddConditionalRoute adds the route and handler if the condition is
// true.
func (app *WebApp) AddConditionalRoute(condition bool, path string, handler RouteHandler) {
	if condition {
		app.AddRoute(path, handler)
	}
}

// StaticRoute adds a new file server route for static assets.
func (app *WebApp) StaticRoute(route string, path string) {
	var err error
	if len(route) == 0 {
		panic("Invalid route: " + route + " -> " + path)
	}
	if len(path) == 0 {
		panic("Invalid path:" + route + " -> " + path)
	} else {
		path, err = filepath.Abs(path)
		if err != nil {
			panic(err)
		}
	}
	app.mux.Handle(route, http.StripPrefix(route, http.FileServer(http.Dir(path))))
	log.Printf("static route %s -> %s added\n", route, path)
}

// GenerateErrorHandler returns a RouteHandler function
func GenerateErrorHandler(status int) ErrorHandlerFn {
	return func(msg, ctype string, w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(status)
		w.Header().Add("content-type", ctype)
		w.Write([]byte(msg))
	}
}

// GenerateTemplateErrorHandler returns a function serving a templated error
func GenerateTemplateErrorHandler(status int, filename string) (hdlr TemplateErrorHandlerFn, err error) {
	tpl, err := CompileTemplate(filename)
	if err != nil {
		return
	}
	hdlr = func(in interface{}, w http.ResponseWriter, r *http.Request) {
		msg, err := BuildTemplate(tpl, in)
		if err != nil {
			log.Printf("error serving template %d %s: %s\n",
				status, filename, err.Error())
			return
		}
		w.WriteHeader(status)
		w.Write(msg)
	}
	return
}

func init() {
	Error400 = GenerateErrorHandler(http.StatusBadRequest)
	Error401 = GenerateErrorHandler(http.StatusUnauthorized)
	Error403 = GenerateErrorHandler(http.StatusForbidden)
	Error404 = GenerateErrorHandler(http.StatusNotFound)
	Error429 = GenerateErrorHandler(429)
	Error500 = GenerateErrorHandler(http.StatusInternalServerError)
	Error501 = GenerateErrorHandler(http.StatusNotImplemented)
	Error502 = GenerateErrorHandler(http.StatusBadGateway)
	Error503 = GenerateErrorHandler(http.StatusServiceUnavailable)

}

// ContentResponder returns the appropriate content-type for the
// request.
func ContentResponder(r *http.Request) string {
	accept := r.Header["Accept"][0]
	if accept == "" {
		return "text/plain"
	} else if accept == "*/*" {
		return "text/plain"
	}
	return accept
}
<|endoftext|>