<commit_before>package lidar_test

import (
	"context"
	"errors"
	"time"

	"code.cloudfoundry.org/lager/lagertest"
	"github.com/concourse/concourse/atc"
	"github.com/concourse/concourse/atc/creds/credsfakes"
	"github.com/concourse/concourse/atc/db"
	"github.com/concourse/concourse/atc/db/dbfakes"
	"github.com/concourse/concourse/atc/db/lock/lockfakes"
	"github.com/concourse/concourse/atc/lidar"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

type Scanner interface {
	Run(ctx context.Context) error
}

var _ = Describe("Scanner", func() {
	var (
		err error

		fakeCheckFactory *dbfakes.FakeCheckFactory
		fakeSecrets      *credsfakes.FakeSecrets

		logger  *lagertest.TestLogger
		scanner Scanner
	)

	BeforeEach(func() {
		fakeCheckFactory = new(dbfakes.FakeCheckFactory)
		fakeSecrets = new(credsfakes.FakeSecrets)

		logger = lagertest.NewTestLogger("test")
		scanner = lidar.NewScanner(
			logger,
			fakeCheckFactory,
			fakeSecrets,
			time.Minute*1,
			time.Minute*1,
			time.Minute*10,
		)
	})

	JustBeforeEach(func() {
		err = scanner.Run(context.TODO())
	})

	Describe("Run", func() {
		var fakeLock *lockfakes.FakeLock

		BeforeEach(func() {
			fakeLock = new(lockfakes.FakeLock)
			fakeCheckFactory.AcquireScanningLockReturns(fakeLock, true, nil)
		})

		Context("when fetching resources fails", func() {
			BeforeEach(func() {
				fakeCheckFactory.ResourcesReturns(nil, errors.New("nope"))
			})

			It("errors", func() {
				Expect(err).To(HaveOccurred())
			})
		})

		Context("when fetching resources succeeds", func() {
			var fakeResource *dbfakes.FakeResource

			BeforeEach(func() {
				fakeResource = new(dbfakes.FakeResource)
				fakeResource.NameReturns("some-name")
				fakeResource.TagsReturns([]string{"tag-a", "tag-b"})
				fakeResource.SourceReturns(atc.Source{"some": "source"})

				fakeCheckFactory.ResourcesReturns([]db.Resource{fakeResource}, nil)
			})

			Context("when fetching resource types fails", func() {
				BeforeEach(func() {
					fakeCheckFactory.ResourceTypesReturns(nil, errors.New("nope"))
				})

				It("errors", func() {
					Expect(err).To(HaveOccurred())
				})
			})

			Context("when fetching resources types succeeds", func() {
				var fakeResourceType *dbfakes.FakeResourceType

				BeforeEach(func() {
					fakeResourceType = new(dbfakes.FakeResourceType)
					fakeResourceType.NameReturns("some-type")
					fakeResourceType.TypeReturns("some-base-type")
					fakeResourceType.TagsReturns([]string{"some-tag"})
					fakeResourceType.SourceReturns(atc.Source{"some": "type-source"})

					fakeCheckFactory.ResourceTypesReturns([]db.ResourceType{fakeResourceType}, nil)
				})

				Context("when the resource parent type is a base type", func() {
					BeforeEach(func() {
						fakeResource.TypeReturns("base-type")
					})

					Context("when the check interval is parseable", func() {
						BeforeEach(func() {
							fakeResource.CheckEveryReturns("10s")
						})

						Context("when the last check end time is within our interval", func() {
							BeforeEach(func() {
								fakeResource.LastCheckEndTimeReturns(time.Now())
							})

							It("does not check", func() {
								Expect(fakeCheckFactory.CreateCheckCallCount()).To(Equal(0))
							})

							It("clears the check error", func() {
								Expect(fakeResource.SetCheckSetupErrorCallCount()).To(Equal(1))
								Expect(fakeResource.SetCheckSetupErrorArgsForCall(0)).To(BeNil())
							})
						})

						Context("when the last check end time is past our interval", func() {
							BeforeEach(func() {
								fakeResource.LastCheckEndTimeReturns(time.Now().Add(-time.Hour))
							})

							It("creates a check", func() {
								Expect(fakeCheckFactory.TryCreateCheckCallCount()).To(Equal(1))
							})

							It("clears the check error", func() {
								Expect(fakeResource.SetCheckSetupErrorCallCount()).To(Equal(1))
								Expect(fakeResource.SetCheckSetupErrorArgsForCall(0)).To(BeNil())
							})

							It("sends a notification for the checker to run", func() {
								Expect(fakeCheckFactory.NotifyCheckerCallCount()).To(Equal(1))
							})

							Context("when try creating a check panic", func() {
								BeforeEach(func() {
									fakeCheckFactory.TryCreateCheckStub = func(context.Context, db.Checkable, db.ResourceTypes, atc.Version, bool) (db.Check, bool, error) {
										panic("something went wrong")
									}
								})

								It("recover from the panic", func() {
									Expect(err).ToNot(HaveOccurred())
									Eventually(fakeResource.SetCheckSetupErrorCallCount).Should(Equal(1))
									Eventually(fakeResource.SetCheckSetupErrorArgsForCall(0).Error).Should(ContainSubstring("something went wrong"))
								})
							})
						})

						Context("when the checkable has a pinned version", func() {
							BeforeEach(func() {
								fakeResource.CurrentPinnedVersionReturns(atc.Version{"some": "version"})
							})

							It("creates a check", func() {
								Expect(fakeCheckFactory.TryCreateCheckCallCount()).To(Equal(1))
								_, _, _, fromVersion, _ := fakeCheckFactory.TryCreateCheckArgsForCall(0)
								Expect(fromVersion).To(Equal(atc.Version{"some": "version"}))
							})

							It("clears the check error", func() {
								Expect(fakeResource.SetCheckSetupErrorCallCount()).To(Equal(1))
								Expect(fakeResource.SetCheckSetupErrorArgsForCall(0)).To(BeNil())
							})

							It("sends a notification for the checker to run", func() {
								Expect(fakeCheckFactory.NotifyCheckerCallCount()).To(Equal(1))
							})
						})

						Context("when the checkable does not have a pinned version", func() {
							BeforeEach(func() {
								fakeResource.CurrentPinnedVersionReturns(nil)
							})

							It("creates a check", func() {
								Expect(fakeCheckFactory.TryCreateCheckCallCount()).To(Equal(1))
								_, _, _, fromVersion, _ := fakeCheckFactory.TryCreateCheckArgsForCall(0)
								Expect(fromVersion).To(BeNil())
							})

							It("clears the check error", func() {
								Expect(fakeResource.SetCheckSetupErrorCallCount()).To(Equal(1))
								Expect(fakeResource.SetCheckSetupErrorArgsForCall(0)).To(BeNil())
							})

							It("sends a notification for the checker to run", func() {
								Expect(fakeCheckFactory.NotifyCheckerCallCount()).To(Equal(1))
							})
						})
					})
				})

				Context("when the resource has a parent type", func() {
					BeforeEach(func() {
						fakeResource.TypeReturns("custom-type")
						fakeResource.PipelineIDReturns(1)
						fakeResourceType.NameReturns("custom-type")
						fakeResourceType.PipelineIDReturns(1)
					})

					Context("when it fails to create a check for parent resource", func() {
						BeforeEach(func() {
							fakeResourceType.CheckEveryReturns("not-a-duration")
						})

						It("sets the check error", func() {
							Expect(fakeResourceType.SetCheckSetupErrorCallCount()).To(Equal(1))
							Expect(fakeResource.SetCheckSetupErrorCallCount()).To(Equal(1))
							err := fakeResource.SetCheckSetupErrorArgsForCall(0)
							Expect(err.Error()).To(ContainSubstring("parent type 'custom-type' error:"))
						})

						It("does not create a check", func() {
							Expect(fakeCheckFactory.TryCreateCheckCallCount()).To(Equal(0))
						})
					})

					Context("when the parent type requires a check", func() {
						BeforeEach(func() {
							fakeResourceType.LastCheckEndTimeReturns(time.Now().Add(-time.Hour))
							fakeResource.LastCheckEndTimeReturns(time.Now().Add(-time.Hour))
						})

						Context("when the parent type has a version", func() {
							BeforeEach(func() {
								fakeResourceType.VersionReturns(atc.Version{"some": "version"})
							})

							It("creates a check for both the parent and the resource", func() {
								Expect(fakeCheckFactory.TryCreateCheckCallCount()).To(Equal(2))

								_, checkable, _, _, manuallyTriggered := fakeCheckFactory.TryCreateCheckArgsForCall(0)
								Expect(checkable).To(Equal(fakeResourceType))
								Expect(manuallyTriggered).To(BeFalse())

								_, checkable, _, _, manuallyTriggered = fakeCheckFactory.TryCreateCheckArgsForCall(1)
								Expect(checkable).To(Equal(fakeResource))
								Expect(manuallyTriggered).To(BeFalse())
							})

							It("sends a notification for the checker to run", func() {
								Expect(fakeCheckFactory.NotifyCheckerCallCount()).To(Equal(1))
							})
						})
					})
				})
			})
		})

		Context("when there are multiple resources that use the same resource type", func() {
			var fakeResource1, fakeResource2 *dbfakes.FakeResource
			var fakeResourceType *dbfakes.FakeResourceType

			BeforeEach(func() {
				fakeResource1 = new(dbfakes.FakeResource)
				fakeResource1.NameReturns("some-name")
				fakeResource1.SourceReturns(atc.Source{"some": "source"})
				fakeResource1.TypeReturns("custom-type")
				fakeResource1.PipelineIDReturns(1)
				fakeResource1.LastCheckEndTimeReturns(time.Now().Add(-time.Hour))

				fakeResource2 = new(dbfakes.FakeResource)
				fakeResource2.NameReturns("some-name")
				fakeResource2.SourceReturns(atc.Source{"some": "source"})
				fakeResource2.TypeReturns("custom-type")
				fakeResource2.PipelineIDReturns(1)
				fakeResource2.LastCheckEndTimeReturns(time.Now().Add(-time.Hour))

				fakeCheckFactory.ResourcesReturns([]db.Resource{fakeResource1, fakeResource2}, nil)

				fakeResourceType = new(dbfakes.FakeResourceType)
				fakeResourceType.NameReturns("custom-type")
				fakeResourceType.PipelineIDReturns(1)
				fakeResourceType.TypeReturns("some-base-type")
				fakeResourceType.SourceReturns(atc.Source{"some": "type-source"})
				fakeResourceType.LastCheckEndTimeReturns(time.Now().Add(-time.Hour))

				fakeCheckFactory.ResourceTypesReturns([]db.ResourceType{fakeResourceType}, nil)
			})

			It("only tries to create a check for the resource type once", func() {
				Expect(fakeCheckFactory.TryCreateCheckCallCount()).To(Equal(3))

				var checked []string
				_, checkable, _, _, _ := fakeCheckFactory.TryCreateCheckArgsForCall(0)
				checked = append(checked, checkable.Name())

				_, checkable, _, _, _ = fakeCheckFactory.TryCreateCheckArgsForCall(1)
				checked = append(checked, checkable.Name())

				_, checkable, _, _, _ = fakeCheckFactory.TryCreateCheckArgsForCall(2)
				checked = append(checked, checkable.Name())

				Expect(checked).To(ConsistOf([]string{fakeResourceType.Name(), fakeResource1.Name(), fakeResource2.Name()}))
			})
		})

		Context("Default with webhook check interval", func() {
			var fakeResource *dbfakes.FakeResource
			BeforeEach(func() {
				fakeResource = new(dbfakes.FakeResource)
				fakeResource.NameReturns("some-name")
				fakeResource.TagsReturns([]string{"tag-a", "tag-b"})
				fakeResource.SourceReturns(atc.Source{"some": "source"})
				fakeResource.TypeReturns("base-type")
				fakeResource.CheckEveryReturns("")
				fakeCheckFactory.ResourcesReturns([]db.Resource{fakeResource}, nil)

			})

			Context("resource has webhook", func() {
				BeforeEach(func() {
					fakeResource.HasWebhookReturns(true)
				})

				Context("last check is 9 minutes ago", func() {
					BeforeEach(func() {
						fakeResource.LastCheckEndTimeReturns(time.Now().Add(-time.Minute * 9))
					})

					It("does not create a check", func() {
						Expect(fakeCheckFactory.TryCreateCheckCallCount()).To(Equal(0))
					})
				})

				Context("last check is 11 minutes ago", func() {
					BeforeEach(func() {
						fakeResource.LastCheckEndTimeReturns(time.Now().Add(-time.Minute * 11))
					})

					It("does not create a check", func() {
						Expect(fakeCheckFactory.TryCreateCheckCallCount()).To(Equal(1))
					})
				})
			})
		})
	})
})
<commit_msg>atc/lidar: fix tests<commit_after>package lidar_test

import (
	"context"
	"errors"
	"time"

	"code.cloudfoundry.org/lager/lagertest"
	"github.com/concourse/concourse/atc"
	"github.com/concourse/concourse/atc/creds/credsfakes"
	"github.com/concourse/concourse/atc/db"
	"github.com/concourse/concourse/atc/db/dbfakes"
	"github.com/concourse/concourse/atc/db/lock/lockfakes"
	"github.com/concourse/concourse/atc/lidar"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

type Scanner interface {
	Run(ctx context.Context) error
}

var _ = Describe("Scanner", func() {
	var (
		err error

		fakeCheckFactory *dbfakes.FakeCheckFactory
		fakeSecrets      *credsfakes.FakeSecrets

		logger  *lagertest.TestLogger
		scanner Scanner
	)

	BeforeEach(func() {
		fakeCheckFactory = new(dbfakes.FakeCheckFactory)
		fakeSecrets = new(credsfakes.FakeSecrets)

		logger = lagertest.NewTestLogger("test")
		scanner = lidar.NewScanner(
			logger,
			fakeCheckFactory,
			fakeSecrets,
			time.Minute*1,
			time.Minute*1,
			time.Minute*10,
		)
	})

	JustBeforeEach(func() {
		err = scanner.Run(context.TODO())
	})

	Describe("Run", func() {
		var fakeLock *lockfakes.FakeLock

		BeforeEach(func() {
			fakeLock = new(lockfakes.FakeLock)
			fakeCheckFactory.AcquireScanningLockReturns(fakeLock, true, nil)
		})

		Context("when fetching resources fails", func() {
			BeforeEach(func() {
				fakeCheckFactory.ResourcesReturns(nil, errors.New("nope"))
			})

			It("errors", func() {
				Expect(err).To(HaveOccurred())
			})
		})

		Context("when fetching resources succeeds", func() {
			var fakeResource *dbfakes.FakeResource

			BeforeEach(func() {
				fakeResource = new(dbfakes.FakeResource)
				fakeResource.NameReturns("some-name")
				fakeResource.TagsReturns([]string{"tag-a", "tag-b"})
				fakeResource.SourceReturns(atc.Source{"some": "source"})

				fakeCheckFactory.ResourcesReturns([]db.Resource{fakeResource}, nil)
			})

			Context("when fetching resource types fails", func() {
				BeforeEach(func() {
					fakeCheckFactory.ResourceTypesReturns(nil, errors.New("nope"))
				})

				It("errors", func() {
					Expect(err).To(HaveOccurred())
				})
			})

			Context("when fetching resources types succeeds", func() {
				var fakeResourceType *dbfakes.FakeResourceType

				BeforeEach(func() {
					fakeResourceType = new(dbfakes.FakeResourceType)
					fakeResourceType.NameReturns("some-type")
					fakeResourceType.TypeReturns("some-base-type")
					fakeResourceType.TagsReturns([]string{"some-tag"})
					fakeResourceType.SourceReturns(atc.Source{"some": "type-source"})

					fakeCheckFactory.ResourceTypesReturns([]db.ResourceType{fakeResourceType}, nil)
				})

				Context("when the resource parent type is a base type", func() {
					BeforeEach(func() {
						fakeResource.TypeReturns("some-type")
					})

					Context("when the check interval is parseable", func() {
						BeforeEach(func() {
							fakeResource.CheckEveryReturns("10s")
						})

						Context("when the last check end time is within our interval", func() {
							BeforeEach(func() {
								fakeResource.LastCheckEndTimeReturns(time.Now())
							})

							It("does not check", func() {
								Expect(fakeCheckFactory.CreateCheckCallCount()).To(Equal(0))
							})

							It("clears the check error", func() {
								Expect(fakeResource.SetCheckSetupErrorCallCount()).To(Equal(1))
								Expect(fakeResource.SetCheckSetupErrorArgsForCall(0)).To(BeNil())
							})
						})

						Context("when the last check end time is past our interval", func() {
							BeforeEach(func() {
								fakeResource.LastCheckEndTimeReturns(time.Now().Add(-time.Hour))
							})

							It("creates a check", func() {
								Expect(fakeCheckFactory.TryCreateCheckCallCount()).To(Equal(2))
							})

							It("clears the check error", func() {
								Expect(fakeResource.SetCheckSetupErrorCallCount()).To(Equal(1))
								Expect(fakeResource.SetCheckSetupErrorArgsForCall(0)).To(BeNil())
							})

							It("sends a notification for the checker to run", func() {
								Expect(fakeCheckFactory.NotifyCheckerCallCount()).To(Equal(1))
							})

							Context("when try creating a check panic", func() {
								BeforeEach(func() {
									fakeCheckFactory.TryCreateCheckStub = func(context.Context, db.Checkable, db.ResourceTypes, atc.Version, bool) (db.Check, bool, error) {
										panic("something went wrong")
									}
								})

								It("recover from the panic", func() {
									Expect(err).ToNot(HaveOccurred())
									Eventually(fakeResource.SetCheckSetupErrorCallCount).Should(Equal(1))
									Eventually(fakeResource.SetCheckSetupErrorArgsForCall(0).Error).Should(ContainSubstring("something went wrong"))
								})
							})
						})

						Context("when the checkable has a pinned version", func() {
							BeforeEach(func() {
								fakeResource.CurrentPinnedVersionReturns(atc.Version{"some": "version"})
							})

							It("creates a check", func() {
								Expect(fakeCheckFactory.TryCreateCheckCallCount()).To(Equal(2))
								_, _, _, fromVersion, _ := fakeCheckFactory.TryCreateCheckArgsForCall(1)
								Expect(fromVersion).To(Equal(atc.Version{"some": "version"}))
							})

							It("clears the check error", func() {
								Expect(fakeResource.SetCheckSetupErrorCallCount()).To(Equal(1))
								Expect(fakeResource.SetCheckSetupErrorArgsForCall(0)).To(BeNil())
							})

							It("sends a notification for the checker to run", func() {
								Expect(fakeCheckFactory.NotifyCheckerCallCount()).To(Equal(1))
							})
						})

						Context("when the checkable does not have a pinned version", func() {
							BeforeEach(func() {
								fakeResource.CurrentPinnedVersionReturns(nil)
							})

							It("creates a check", func() {
								Expect(fakeCheckFactory.TryCreateCheckCallCount()).To(Equal(2))
								_, _, _, fromVersion, _ := fakeCheckFactory.TryCreateCheckArgsForCall(0)
								Expect(fromVersion).To(BeNil())
							})

							It("clears the check error", func() {
								Expect(fakeResource.SetCheckSetupErrorCallCount()).To(Equal(1))
								Expect(fakeResource.SetCheckSetupErrorArgsForCall(0)).To(BeNil())
							})

							It("sends a notification for the checker to run", func() {
								Expect(fakeCheckFactory.NotifyCheckerCallCount()).To(Equal(1))
							})
						})
					})
				})

				Context("when the resource has a parent type", func() {
					BeforeEach(func() {
						fakeResource.TypeReturns("custom-type")
						fakeResource.PipelineIDReturns(1)
						fakeResourceType.NameReturns("custom-type")
						fakeResourceType.PipelineIDReturns(1)
					})

					Context("when it fails to create a check for parent resource", func() {
						BeforeEach(func() {
							fakeResourceType.CheckEveryReturns("not-a-duration")
						})

						It("sets the check error", func() {
							Expect(fakeResourceType.SetCheckSetupErrorCallCount()).To(Equal(1))
							Expect(fakeResource.SetCheckSetupErrorCallCount()).To(Equal(1))
							err := fakeResource.SetCheckSetupErrorArgsForCall(0)
							Expect(err.Error()).To(ContainSubstring("parent type 'custom-type' error:"))
						})

						It("does not create a check", func() {
							Expect(fakeCheckFactory.TryCreateCheckCallCount()).To(Equal(0))
						})
					})

					Context("when the parent type requires a check", func() {
						BeforeEach(func() {
							fakeResourceType.LastCheckEndTimeReturns(time.Now().Add(-time.Hour))
							fakeResource.LastCheckEndTimeReturns(time.Now().Add(-time.Hour))
						})

						Context("when the parent type has a version", func() {
							BeforeEach(func() {
								fakeResourceType.VersionReturns(atc.Version{"some": "version"})
							})

							It("creates a check for both the parent and the resource", func() {
								Expect(fakeCheckFactory.TryCreateCheckCallCount()).To(Equal(2))

								_, checkable, _, _, manuallyTriggered := fakeCheckFactory.TryCreateCheckArgsForCall(0)
								Expect(checkable).To(Equal(fakeResourceType))
								Expect(manuallyTriggered).To(BeFalse())

								_, checkable, _, _, manuallyTriggered = fakeCheckFactory.TryCreateCheckArgsForCall(1)
								Expect(checkable).To(Equal(fakeResource))
								Expect(manuallyTriggered).To(BeFalse())
							})

							It("sends a notification for the checker to run", func() {
								Expect(fakeCheckFactory.NotifyCheckerCallCount()).To(Equal(1))
							})
						})
					})
				})
			})
		})

		Context("when there are multiple resources that use the same resource type", func() {
			var fakeResource1, fakeResource2 *dbfakes.FakeResource
			var fakeResourceType *dbfakes.FakeResourceType

			BeforeEach(func() {
				fakeResource1 = new(dbfakes.FakeResource)
				fakeResource1.NameReturns("some-name")
				fakeResource1.SourceReturns(atc.Source{"some": "source"})
				fakeResource1.TypeReturns("custom-type")
				fakeResource1.PipelineIDReturns(1)
				fakeResource1.LastCheckEndTimeReturns(time.Now().Add(-time.Hour))

				fakeResource2 = new(dbfakes.FakeResource)
				fakeResource2.NameReturns("some-name")
				fakeResource2.SourceReturns(atc.Source{"some": "source"})
				fakeResource2.TypeReturns("custom-type")
				fakeResource2.PipelineIDReturns(1)
				fakeResource2.LastCheckEndTimeReturns(time.Now().Add(-time.Hour))

				fakeCheckFactory.ResourcesReturns([]db.Resource{fakeResource1, fakeResource2}, nil)

				fakeResourceType = new(dbfakes.FakeResourceType)
				fakeResourceType.NameReturns("custom-type")
				fakeResourceType.PipelineIDReturns(1)
				fakeResourceType.TypeReturns("some-base-type")
				fakeResourceType.SourceReturns(atc.Source{"some": "type-source"})
				fakeResourceType.LastCheckEndTimeReturns(time.Now().Add(-time.Hour))

				fakeCheckFactory.ResourceTypesReturns([]db.ResourceType{fakeResourceType}, nil)
			})

			It("only tries to create a check for the resource type once", func() {
				Expect(fakeCheckFactory.TryCreateCheckCallCount()).To(Equal(3))

				var checked []string
				_, checkable, _, _, _ := fakeCheckFactory.TryCreateCheckArgsForCall(0)
				checked = append(checked, checkable.Name())

				_, checkable, _, _, _ = fakeCheckFactory.TryCreateCheckArgsForCall(1)
				checked = append(checked, checkable.Name())

				_, checkable, _, _, _ = fakeCheckFactory.TryCreateCheckArgsForCall(2)
				checked = append(checked, checkable.Name())

				Expect(checked).To(ConsistOf([]string{fakeResourceType.Name(), fakeResource1.Name(), fakeResource2.Name()}))
			})
		})

		Context("Default with webhook check interval", func() {
			var fakeResource *dbfakes.FakeResource
			BeforeEach(func() {
				fakeResource = new(dbfakes.FakeResource)
				fakeResource.NameReturns("some-name")
				fakeResource.TagsReturns([]string{"tag-a", "tag-b"})
				fakeResource.SourceReturns(atc.Source{"some": "source"})
				fakeResource.TypeReturns("base-type")
				fakeResource.CheckEveryReturns("")
				fakeCheckFactory.ResourcesReturns([]db.Resource{fakeResource}, nil)

			})

			Context("resource has webhook", func() {
				BeforeEach(func() {
					fakeResource.HasWebhookReturns(true)
				})

				Context("last check is 9 minutes ago", func() {
					BeforeEach(func() {
						fakeResource.LastCheckEndTimeReturns(time.Now().Add(-time.Minute * 9))
					})

					It("does not create a check", func() {
						Expect(fakeCheckFactory.TryCreateCheckCallCount()).To(Equal(0))
					})
				})

				Context("last check is 11 minutes ago", func() {
					BeforeEach(func() {
						fakeResource.LastCheckEndTimeReturns(time.Now().Add(-time.Minute * 11))
					})

					It("does not create a check", func() {
						Expect(fakeCheckFactory.TryCreateCheckCallCount()).To(Equal(1))
					})
				})
			})
		})
	})
})
<|endoftext|>


<commit_before>// Copyright © 2017 The Kubicorn Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package logger

import (
	"bytes"
	"fmt"
	"regexp"
	"strings"
	"testing"
)

const (
	format    = "%v, %v, %v, all eyes on me!"
	formatExp = `^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.* \[%s\]  \d, \d, \d, all eyes on me!`
)

var (
	a = []interface{}{1, 2, 3}
)

func TestMain(m *testing.M) {
	TestMode = true

	m.Run()
}

func ExampleTestLog() {
	Log(format, a...)
	// Output: 1, 2, 3, all eyes on me!
}

func TestLog(t *testing.T) {
	e := fmt.Sprintf(format, a...)
	g := captureLoggerOutput(Log, format, a)

	if strings.Compare(e, g) != 0 {
		t.Fatalf("Log should produce '%v' but produces: %v", e, g)
	}
}

func TestAlways(t *testing.T) {
	e, err := regexp.Compile(fmt.Sprintf(formatExp, AlwaysLabel))
	g := captureLoggerOutput(Always, format, a)

	if err != nil {
		t.Fatalf("Failed to compile regexp '%v': %v", e.String(), err)
	}

	if !e.MatchString(g) {
		t.Fatalf("Always should produce a pattern '%v' but produces: %v", e.String(), g)
	}
}

func TestCritical(t *testing.T) {
	Level = 1

	e, err := regexp.Compile(fmt.Sprintf(formatExp, CriticalLabel))
	g := captureLoggerOutput(Critical, format, a)

	if err != nil {
		t.Fatalf("Failed to compile regexp '%v': %v", e.String(), err)
	}

	if !e.MatchString(g) {
		t.Fatalf("Critical should produce a pattern '%v' but produces: %v", e.String(), g)
	}
}

func TestInfo(t *testing.T) {
	Level = 3

	e, err := regexp.Compile(fmt.Sprintf(formatExp, InfoLabel))
	g := captureLoggerOutput(Info, format, a)

	if err != nil {
		t.Fatalf("Failed to compile regexp '%v': %v", e.String(), err)
	}

	if !e.MatchString(g) {
		t.Fatalf("Info should produce a pattern '%v' but produces: %v", e.String(), g)
	}
}

func TestDebug(t *testing.T) {
	Level = 4

	e, err := regexp.Compile(fmt.Sprintf(formatExp, DebugLabel))
	g := captureLoggerOutput(Debug, format, a)

	if err != nil {
		t.Fatalf("Failed to compile regexp '%v': %v", e.String(), err)
	}

	if !e.MatchString(g) {
		t.Fatalf("Info should produce a pattern '%v' but produces: %v", e.String(), g)
	}
}

func TestWarning(t *testing.T) {
	Level = 2

	e, err := regexp.Compile(fmt.Sprintf(formatExp, WarningLabel))
	g := captureLoggerOutput(Warning, format, a)

	if err != nil {
		t.Fatalf("Failed to compile regexp '%v': %v", e.String(), err)
	}

	if !e.MatchString(g) {
		t.Fatalf("Info should produce a pattern '%v' but produces: %v", e.String(), g)
	}
}

func captureLoggerOutput(l Logger, format string, a []interface{}) string {
	b := new(bytes.Buffer)
	l(format, append(a, b)...)
	return b.String()
}
<commit_msg>logger: implement test for success function<commit_after>// Copyright © 2017 The Kubicorn Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package logger

import (
	"bytes"
	"fmt"
	"regexp"
	"strings"
	"testing"
)

const (
	format    = "%v, %v, %v, all eyes on me!"
	formatExp = `^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.* \[%s\]  \d, \d, \d, all eyes on me!`
)

var (
	a = []interface{}{1, 2, 3}
)

func TestMain(m *testing.M) {
	TestMode = true

	m.Run()
}

func ExampleTestLog() {
	Log(format, a...)
	// Output: 1, 2, 3, all eyes on me!
}

func TestLog(t *testing.T) {
	e := fmt.Sprintf(format, a...)
	g := captureLoggerOutput(Log, format, a)

	if strings.Compare(e, g) != 0 {
		t.Fatalf("Log should produce '%v' but produces: %v", e, g)
	}
}

func TestAlways(t *testing.T) {
	e, err := regexp.Compile(fmt.Sprintf(formatExp, AlwaysLabel))
	g := captureLoggerOutput(Always, format, a)

	if err != nil {
		t.Fatalf("Failed to compile regexp '%v': %v", e.String(), err)
	}

	if !e.MatchString(g) {
		t.Fatalf("Always should produce a pattern '%v' but produces: %v", e.String(), g)
	}
}

func TestCritical(t *testing.T) {
	Level = 1

	e, err := regexp.Compile(fmt.Sprintf(formatExp, CriticalLabel))
	g := captureLoggerOutput(Critical, format, a)

	if err != nil {
		t.Fatalf("Failed to compile regexp '%v': %v", e.String(), err)
	}

	if !e.MatchString(g) {
		t.Fatalf("Critical should produce a pattern '%v' but produces: %v", e.String(), g)
	}
}

func TestInfo(t *testing.T) {
	Level = 3

	e, err := regexp.Compile(fmt.Sprintf(formatExp, InfoLabel))
	g := captureLoggerOutput(Info, format, a)

	if err != nil {
		t.Fatalf("Failed to compile regexp '%v': %v", e.String(), err)
	}

	if !e.MatchString(g) {
		t.Fatalf("Info should produce a pattern '%v' but produces: %v", e.String(), g)
	}
}

func TestSuccess(t *testing.T) {
	Level = 3

	e, err := regexp.Compile(fmt.Sprintf(formatExp, SuccessLabel))
	g := captureLoggerOutput(Success, format, a)

	if err != nil {
		t.Fatalf("Failed to compile regexp '%v': %v", e.String(), err)
	}

	if !e.MatchString(g) {
		t.Fatalf("Success should produce a pattern '%v' but produces: %v", e.String(), g)
	}
}

func TestDebug(t *testing.T) {
	Level = 4

	e, err := regexp.Compile(fmt.Sprintf(formatExp, DebugLabel))
	g := captureLoggerOutput(Debug, format, a)

	if err != nil {
		t.Fatalf("Failed to compile regexp '%v': %v", e.String(), err)
	}

	if !e.MatchString(g) {
		t.Fatalf("Info should produce a pattern '%v' but produces: %v", e.String(), g)
	}
}

func TestWarning(t *testing.T) {
	Level = 2

	e, err := regexp.Compile(fmt.Sprintf(formatExp, WarningLabel))
	g := captureLoggerOutput(Warning, format, a)

	if err != nil {
		t.Fatalf("Failed to compile regexp '%v': %v", e.String(), err)
	}

	if !e.MatchString(g) {
		t.Fatalf("Info should produce a pattern '%v' but produces: %v", e.String(), g)
	}
}

func captureLoggerOutput(l Logger, format string, a []interface{}) string {
	b := new(bytes.Buffer)
	l(format, append(a, b)...)
	return b.String()
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"log"
)

func newCommand(args []string) (*Command, error) {
	const argsError = `use "build", "run" or "test".`
	if len(args) < 2 {
		return nil, errors.New(argsError)
	}

	var (
		c   Command
		err error
	)
	switch args[1] {
	case "one":
		err = newOneCommand(&c)
	case "two":
		err = newTwoCommand(&c)
	case "three":
		err = newThreeCommand(&c)
	default:
		err = errors.New(argsError)
	}

	if err != nil {
		return nil, fmt.Errorf("command: %v", err)
	}

	return &c, nil
}

type Command struct {
	Kind int
}

const (
	OneCommand = iota
	TwoCommand
	ThreeCommand
)

func newOneCommand(c *Command) error {
	log.SetPrefix("one: ")

	if c == nil {
		return errors.New("nil command")
	}

	c.Kind = OneCommand

	return nil
}

func newTwoCommand(c *Command) error {
	log.SetPrefix("two: ")

	if c == nil {
		return errors.New("nil command")
	}

	c.Kind = TwoCommand

	return nil
}

func newThreeCommand(c *Command) error {
	log.SetPrefix("three: ")

	if c == nil {
		return errors.New("nil command")
	}

	c.Kind = ThreeCommand

	return nil
}
<commit_msg>command: change argsError<commit_after>package main

import (
	"errors"
	"fmt"
	"log"
)

func newCommand(args []string) (*Command, error) {
	const argsError = `use "one", "two" or "three".`
	if len(args) < 2 {
		return nil, errors.New(argsError)
	}

	var (
		c   Command
		err error
	)
	switch args[1] {
	case "one":
		err = newOneCommand(&c)
	case "two":
		err = newTwoCommand(&c)
	case "three":
		err = newThreeCommand(&c)
	default:
		err = errors.New(argsError)
	}

	if err != nil {
		return nil, fmt.Errorf("command: %v", err)
	}

	return &c, nil
}

type Command struct {
	Kind int
}

const (
	OneCommand = iota
	TwoCommand
	ThreeCommand
)

func newOneCommand(c *Command) error {
	log.SetPrefix("one: ")

	if c == nil {
		return errors.New("nil command")
	}

	c.Kind = OneCommand

	return nil
}

func newTwoCommand(c *Command) error {
	log.SetPrefix("two: ")

	if c == nil {
		return errors.New("nil command")
	}

	c.Kind = TwoCommand

	return nil
}

func newThreeCommand(c *Command) error {
	log.SetPrefix("three: ")

	if c == nil {
		return errors.New("nil command")
	}

	c.Kind = ThreeCommand

	return nil
}
<|endoftext|>


<commit_before>package command

import (
	"bufio"
	"io"
	"os/exec"
)

//Cmd is a wrapper around Cmd
type Cmd struct {
	StdoutChannel chan string
	StderrChannel chan string
	Cmd           *exec.Cmd
}

func checkError(err error) {
	if err != nil {
		panic(err)
	}
}

// Command returns a an executil Cmd struct with
// an exec.Cmd struct embedded in it
func Command(name string, arg ...string) *Cmd {
	c := new(Cmd)
	// set the exec.Cmd
	c.Cmd = exec.Command(name, arg...)
	return c
}

// CombinedOutput wrapper for exec.Cmd.CombinedOutput()
func (c *Cmd) CombinedOutput() ([]byte, error) {
	return c.Cmd.CombinedOutput()
}
func (c *Cmd) MustCombinedOutput() []byte {
	out, err := c.CombinedOutput()
	checkError(err)
	return out
}

// Output wrapper for exec.Cmd.Output()
func (c *Cmd) Output() ([]byte, error) {
	return c.Cmd.Output()
}
func (c *Cmd) MustOutput() []byte {
	out, err := c.Output()
	checkError(err)
	return out
}

// Run wrapper for exec.Cmd.Run()
func (c *Cmd) Run() error {
	return c.Cmd.Run()
}
func (c *Cmd) MustRun() {
	checkError(c.Run())
}

// Start wrapper for exec.Cmd.Start()
func (c *Cmd) Start() error {
	// go routines to scan command out and err
	err := c.createPipeScanners()
	checkError(err)

	return c.Cmd.Start()
}
func (c *Cmd) MustStart() {
	checkError(c.Start())
}

// StderrPipe wrapper for exec.Cmd.StderrPipe()
func (c *Cmd) StderrPipe() (io.ReadCloser, error) {
	return c.Cmd.StderrPipe()
}
func (c *Cmd) MustStderrPipe() io.ReadCloser {
	pipe, err := c.StderrPipe()
	checkError(err)
	return pipe
}

// StdinPipe wrapper for exec.Cmd.StdinPipe()
func (c *Cmd) StdinPipe() (io.WriteCloser, error) {
	return c.Cmd.StdinPipe()
}
func (c *Cmd) MustStdinPipe() io.WriteCloser {
	pipe, err := c.StdinPipe()
	checkError(err)
	return pipe
}

// StdoutPipe wrapper for exec.Cmd.StdoutPipe()
func (c *Cmd) StdoutPipe() (io.ReadCloser, error) {
	return c.Cmd.StdoutPipe()
}
func (c *Cmd) MustStdoutPipe() io.ReadCloser {
	pipe, err := c.StdoutPipe()
	checkError(err)
	return pipe
}

// Wait wrapper for exec.Cmd.Wait()
func (c *Cmd) Wait() error {
	return c.Cmd.Wait()
}
func (c *Cmd) MustWait() {
	checkError(c.Wait())
}

// Create stdout, and stderr pipes for given *Cmd
// Only works with cmd.Start()
func (c *Cmd) createPipeScanners() error {
	if c.StdoutChannel != nil {
		stdout, err := c.Cmd.StdoutPipe()
		if err != nil {
			return err
		}

		outScanner := bufio.NewScanner(stdout)

		go func() {
			for outScanner.Scan() {
				c.StdoutChannel <- outScanner.Text()
			}
		}()
	}

	if c.StderrChannel != nil {
		stderr, err := c.Cmd.StderrPipe()
		if err != nil {
			return err
		}

		errScanner := bufio.NewScanner(stderr)

		// Scan for text
		go func() {
			for errScanner.Scan() {
				c.StderrChannel <- errScanner.Text()
			}
		}()
	}

	return nil
}
<commit_msg>Directly embed the `*exec.Cmd` so that we have access to its fields like `Dir`, `Env`, etc.<commit_after>package command

import (
	"bufio"
	"io"
	"os/exec"
)

//Cmd is a wrapper around Cmd
type Cmd struct {
	StdoutChannel chan string
	StderrChannel chan string
	*exec.Cmd
}

func checkError(err error) {
	if err != nil {
		panic(err)
	}
}

// Command returns a an executil Cmd struct with
// an exec.Cmd struct embedded in it
func Command(name string, arg ...string) *Cmd {
	c := new(Cmd)
	// set the exec.Cmd
	c.Cmd = exec.Command(name, arg...)
	return c
}

// CombinedOutput wrapper for exec.Cmd.CombinedOutput()
func (c *Cmd) CombinedOutput() ([]byte, error) {
	return c.Cmd.CombinedOutput()
}
func (c *Cmd) MustCombinedOutput() []byte {
	out, err := c.CombinedOutput()
	checkError(err)
	return out
}

// Output wrapper for exec.Cmd.Output()
func (c *Cmd) Output() ([]byte, error) {
	return c.Cmd.Output()
}
func (c *Cmd) MustOutput() []byte {
	out, err := c.Output()
	checkError(err)
	return out
}

// Run wrapper for exec.Cmd.Run()
func (c *Cmd) Run() error {
	return c.Cmd.Run()
}
func (c *Cmd) MustRun() {
	checkError(c.Run())
}

// Start wrapper for exec.Cmd.Start()
func (c *Cmd) Start() error {
	// go routines to scan command out and err
	err := c.createPipeScanners()
	checkError(err)

	return c.Cmd.Start()
}
func (c *Cmd) MustStart() {
	checkError(c.Start())
}

// StderrPipe wrapper for exec.Cmd.StderrPipe()
func (c *Cmd) StderrPipe() (io.ReadCloser, error) {
	return c.Cmd.StderrPipe()
}
func (c *Cmd) MustStderrPipe() io.ReadCloser {
	pipe, err := c.StderrPipe()
	checkError(err)
	return pipe
}

// StdinPipe wrapper for exec.Cmd.StdinPipe()
func (c *Cmd) StdinPipe() (io.WriteCloser, error) {
	return c.Cmd.StdinPipe()
}
func (c *Cmd) MustStdinPipe() io.WriteCloser {
	pipe, err := c.StdinPipe()
	checkError(err)
	return pipe
}

// StdoutPipe wrapper for exec.Cmd.StdoutPipe()
func (c *Cmd) StdoutPipe() (io.ReadCloser, error) {
	return c.Cmd.StdoutPipe()
}
func (c *Cmd) MustStdoutPipe() io.ReadCloser {
	pipe, err := c.StdoutPipe()
	checkError(err)
	return pipe
}

// Wait wrapper for exec.Cmd.Wait()
func (c *Cmd) Wait() error {
	return c.Cmd.Wait()
}
func (c *Cmd) MustWait() {
	checkError(c.Wait())
}

// Create stdout, and stderr pipes for given *Cmd
// Only works with cmd.Start()
func (c *Cmd) createPipeScanners() error {
	if c.StdoutChannel != nil {
		stdout, err := c.Cmd.StdoutPipe()
		if err != nil {
			return err
		}

		outScanner := bufio.NewScanner(stdout)

		go func() {
			for outScanner.Scan() {
				c.StdoutChannel <- outScanner.Text()
			}
		}()
	}

	if c.StderrChannel != nil {
		stderr, err := c.Cmd.StderrPipe()
		if err != nil {
			return err
		}

		errScanner := bufio.NewScanner(stderr)

		// Scan for text
		go func() {
			for errScanner.Scan() {
				c.StderrChannel <- errScanner.Text()
			}
		}()
	}

	return nil
}
<|endoftext|>


<commit_before>package commands

import (
	"github.com/jingweno/gh/git"
	"github.com/jingweno/gh/github"
	"github.com/jingweno/gh/utils"
)

var cmdRemote = &Command{
	Run:          remote,
	GitExtension: true,
	Usage:        "remote [-p] OPTIONS USER[/REPOSITORY]",
	Short:        "View and manage a set of remote repositories",
}

/**
  $ gh remote add jingweno
  > git remote add jingweno git://github.com/jingweno/THIS_REPO.git

  $ gh remote add -p jingweno
  > git remote add jingweno git@github.com:jingweno/THIS_REPO.git

  $ gh remote add origin
  > git remote add origin
  git://github.com/YOUR_LOGIN/THIS_REPO.git
**/
func remote(command *Command, args []string) {
	if len(args) >= 1 && (args[0] == "add" || args[0] == "set-url") {
		args = transformRemoteArgs(args)
	}

	err := git.ExecRemote(args)
	utils.Check(err)
}

func transformRemoteArgs(args []string) (newArgs []string) {
	args, isPriavte := parseRemotePrivateFlag(args)
	newArgs, owner := removeItem(args, len(args)-1)

	gh := github.New()
	url := gh.ExpandRemoteUrl(owner, isPriavte)

	return append(newArgs, owner, url)
}

func parseRemotePrivateFlag(args []string) ([]string, bool) {
	for i, arg := range args {
		if arg == "-p" {
			args, _ = removeItem(args, i)
			return args, true
		}
	}

	return args, false
}
<commit_msg>Only transform args when it's more than two<commit_after>package commands

import (
	"github.com/jingweno/gh/git"
	"github.com/jingweno/gh/github"
	"github.com/jingweno/gh/utils"
)

var cmdRemote = &Command{
	Run:          remote,
	GitExtension: true,
	Usage:        "remote [-p] OPTIONS USER[/REPOSITORY]",
	Short:        "View and manage a set of remote repositories",
}

/**
  $ gh remote add jingweno
  > git remote add jingweno git://github.com/jingweno/THIS_REPO.git

  $ gh remote add -p jingweno
  > git remote add jingweno git@github.com:jingweno/THIS_REPO.git

  $ gh remote add origin
  > git remote add origin
  git://github.com/YOUR_LOGIN/THIS_REPO.git
**/
func remote(command *Command, args []string) {
	if len(args) >= 2 && (args[0] == "add" || args[0] == "set-url") {
		args = transformRemoteArgs(args)
	}

	err := git.ExecRemote(args)
	utils.Check(err)
}

func transformRemoteArgs(args []string) (newArgs []string) {
	args, isPriavte := parseRemotePrivateFlag(args)
	newArgs, owner := removeItem(args, len(args)-1)

	gh := github.New()
	url := gh.ExpandRemoteUrl(owner, isPriavte)

	return append(newArgs, owner, url)
}

func parseRemotePrivateFlag(args []string) ([]string, bool) {
	for i, arg := range args {
		if arg == "-p" {
			args, _ = removeItem(args, i)
			return args, true
		}
	}

	return args, false
}
<|endoftext|>


<commit_before>// Copyright 2014, Google, Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style license
// that can be found in the LICENSE file in the root of the source
// tree.

package layers

import (
	"github.com/google/gopacket"
	"testing"
	"net"
)

const (
	ipv4UDPChecksum = uint16(0xbc5f) // Wireshark
	ipv6UDPChecksumWithIPv6DstOpts = uint16(0x4d21) // Wireshark
)

func createIPv4ChecksumTestLayer() (ip4 *IPv4) {
	ip4 = &IPv4{}
	ip4.Version = 4
	ip4.TTL = 64
	ip4.SrcIP = net.ParseIP("192.0.2.1")
	ip4.DstIP = net.ParseIP("198.51.100.1")
	return
}

func createIPv6ChecksumTestLayer() (ip6 *IPv6) {
	ip6 = &IPv6{}
	ip6.Version = 6
	ip6.NextHeader = IPProtocolNoNextHeader
	ip6.HopLimit = 64
	ip6.SrcIP = net.ParseIP("2001:db8::1")
	ip6.DstIP = net.ParseIP("2001:db8::2")
	return
}

func createIPv6DestinationChecksumTestLayer() (dst *IPv6Destination) {
	tlv := &IPv6DestinationOption{}
	tlv.OptionType = 0x01 //PadN
	tlv.OptionData = []byte{0x00, 0x00, 0x00, 0x00}
	dst = &IPv6Destination{}
	dst.Options = append(dst.Options, *tlv)
	dst.NextHeader = IPProtocolNoNextHeader
	return
}

func createUDPChecksumTestLayer() (udp *UDP) {
	udp = &UDP{}
	udp.SrcPort = UDPPort(12345)
	udp.DstPort = UDPPort(9999)
	return
}

func TestIPv4UDPChecksum(t *testing.T) {
	var serialize []gopacket.SerializableLayer = make([]gopacket.SerializableLayer, 0, 2)
	var u *UDP
	var err error

	ip4 := createIPv4ChecksumTestLayer()
	ip4.Protocol = IPProtocolUDP
	serialize = append(serialize, ip4)

	udp := createUDPChecksumTestLayer()
	udp.SetNetworkLayerForChecksum(ip4)
	serialize = append(serialize, udp)

	buf := gopacket.NewSerializeBuffer()
	opts := gopacket.SerializeOptions{FixLengths: true, ComputeChecksums: true}
	err = gopacket.SerializeLayers(buf, opts, serialize...)
	if err != nil {
		t.Fatal(err)
	}

	p := gopacket.NewPacket(buf.Bytes(), LinkTypeRaw, gopacket.Default)
	if p.ErrorLayer() != nil {
		t.Fatal("Failed to decode packet:", p.ErrorLayer().Error())
	}
	if l, ok := p.Layer(LayerTypeUDP).(*UDP); !ok {
		t.Fatal("No UDP layer type found in packet")
	} else {
		u = l
	}

	got := u.Checksum
	want := ipv4UDPChecksum
	if got != want {
		t.Errorf("Bad checksum:\ngot:\n%#v\n\nwant:\n%#v\n\n", got, want)
	}
}

func TestIPv6UDPChecksumWithIPv6DstOpts(t *testing.T) {
	var serialize []gopacket.SerializableLayer = make([]gopacket.SerializableLayer, 0, 3)
	var u *UDP
	var err error

	ip6 := createIPv6ChecksumTestLayer()
	ip6.NextHeader = IPProtocolIPv6Destination
	serialize = append(serialize, ip6)

	dst := createIPv6DestinationChecksumTestLayer()
	dst.NextHeader = IPProtocolUDP
	serialize = append(serialize, dst)

	udp := createUDPChecksumTestLayer()
	udp.SetNetworkLayerForChecksum(ip6)
	serialize = append(serialize, udp)

	buf := gopacket.NewSerializeBuffer()
	opts := gopacket.SerializeOptions{FixLengths: true, ComputeChecksums: true}
	err = gopacket.SerializeLayers(buf, opts, serialize...)
	if err != nil {
		t.Fatal(err)
	}

	p := gopacket.NewPacket(buf.Bytes(), LinkTypeRaw, gopacket.Default)
	if p.ErrorLayer() != nil {
		t.Fatal("Failed to decode packet:", p.ErrorLayer().Error())
	}
	if l, ok := p.Layer(LayerTypeUDP).(*UDP); !ok {
		t.Fatal("No UDP layer type found in packet")
	} else {
		u = l
	}

	got := u.Checksum
	want := ipv6UDPChecksumWithIPv6DstOpts
	if got != want {
		t.Errorf("Bad checksum:\ngot:\n%#v\n\nwant:\n%#v\n\n", got, want)
	}
}
<commit_msg>tcpip_test: add missing checkLayers()<commit_after>// Copyright 2014, Google, Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style license
// that can be found in the LICENSE file in the root of the source
// tree.

package layers

import (
	"github.com/google/gopacket"
	"testing"
	"net"
)

const (
	ipv4UDPChecksum = uint16(0xbc5f) // Wireshark
	ipv6UDPChecksumWithIPv6DstOpts = uint16(0x4d21) // Wireshark
)

func createIPv4ChecksumTestLayer() (ip4 *IPv4) {
	ip4 = &IPv4{}
	ip4.Version = 4
	ip4.TTL = 64
	ip4.SrcIP = net.ParseIP("192.0.2.1")
	ip4.DstIP = net.ParseIP("198.51.100.1")
	return
}

func createIPv6ChecksumTestLayer() (ip6 *IPv6) {
	ip6 = &IPv6{}
	ip6.Version = 6
	ip6.NextHeader = IPProtocolNoNextHeader
	ip6.HopLimit = 64
	ip6.SrcIP = net.ParseIP("2001:db8::1")
	ip6.DstIP = net.ParseIP("2001:db8::2")
	return
}

func createIPv6DestinationChecksumTestLayer() (dst *IPv6Destination) {
	tlv := &IPv6DestinationOption{}
	tlv.OptionType = 0x01 //PadN
	tlv.OptionData = []byte{0x00, 0x00, 0x00, 0x00}
	dst = &IPv6Destination{}
	dst.Options = append(dst.Options, *tlv)
	dst.NextHeader = IPProtocolNoNextHeader
	return
}

func createUDPChecksumTestLayer() (udp *UDP) {
	udp = &UDP{}
	udp.SrcPort = UDPPort(12345)
	udp.DstPort = UDPPort(9999)
	return
}

func TestIPv4UDPChecksum(t *testing.T) {
	var serialize []gopacket.SerializableLayer = make([]gopacket.SerializableLayer, 0, 2)
	var u *UDP
	var err error

	ip4 := createIPv4ChecksumTestLayer()
	ip4.Protocol = IPProtocolUDP
	serialize = append(serialize, ip4)

	udp := createUDPChecksumTestLayer()
	udp.SetNetworkLayerForChecksum(ip4)
	serialize = append(serialize, udp)

	buf := gopacket.NewSerializeBuffer()
	opts := gopacket.SerializeOptions{FixLengths: true, ComputeChecksums: true}
	err = gopacket.SerializeLayers(buf, opts, serialize...)
	if err != nil {
		t.Fatal(err)
	}

	p := gopacket.NewPacket(buf.Bytes(), LinkTypeRaw, gopacket.Default)
	if p.ErrorLayer() != nil {
		t.Fatal("Failed to decode packet:", p.ErrorLayer().Error())
	}
	checkLayers(p, []gopacket.LayerType{LayerTypeIPv4, LayerTypeUDP}, t)

	if l, ok := p.Layer(LayerTypeUDP).(*UDP); !ok {
		t.Fatal("No UDP layer type found in packet")
	} else {
		u = l
	}
	got := u.Checksum
	want := ipv4UDPChecksum
	if got != want {
		t.Errorf("Bad checksum:\ngot:\n%#v\n\nwant:\n%#v\n\n", got, want)
	}
}

func TestIPv6UDPChecksumWithIPv6DstOpts(t *testing.T) {
	var serialize []gopacket.SerializableLayer = make([]gopacket.SerializableLayer, 0, 3)
	var u *UDP
	var err error

	ip6 := createIPv6ChecksumTestLayer()
	ip6.NextHeader = IPProtocolIPv6Destination
	serialize = append(serialize, ip6)

	dst := createIPv6DestinationChecksumTestLayer()
	dst.NextHeader = IPProtocolUDP
	serialize = append(serialize, dst)

	udp := createUDPChecksumTestLayer()
	udp.SetNetworkLayerForChecksum(ip6)
	serialize = append(serialize, udp)

	buf := gopacket.NewSerializeBuffer()
	opts := gopacket.SerializeOptions{FixLengths: true, ComputeChecksums: true}
	err = gopacket.SerializeLayers(buf, opts, serialize...)
	if err != nil {
		t.Fatal(err)
	}

	p := gopacket.NewPacket(buf.Bytes(), LinkTypeRaw, gopacket.Default)
	if p.ErrorLayer() != nil {
		t.Fatal("Failed to decode packet:", p.ErrorLayer().Error())
	}
	checkLayers(p, []gopacket.LayerType{LayerTypeIPv6, LayerTypeIPv6Destination, LayerTypeUDP}, t)

	if l, ok := p.Layer(LayerTypeUDP).(*UDP); !ok {
		t.Fatal("No UDP layer type found in packet")
	} else {
		u = l
	}
	got := u.Checksum
	want := ipv6UDPChecksumWithIPv6DstOpts
	if got != want {
		t.Errorf("Bad checksum:\ngot:\n%#v\n\nwant:\n%#v\n\n", got, want)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lease

import (
	"container/list"
	"fmt"
	"os"

	"github.com/jacobsa/fuse/fsutil"
	"github.com/jacobsa/gcloud/syncutil"
)

// A type that manages read and read/write leases for anonymous temporary files.
//
// Safe for concurrent access. Must be created with NewFileLeaser.
type FileLeaser struct {
	/////////////////////////
	// Constant data
	/////////////////////////

	dir   string
	limit int64

	/////////////////////////
	// Mutable state
	/////////////////////////

	// A lock that guards the mutable state in this struct. Usually this is used
	// only for light weight operations, but while evicting it may require
	// waiting on a goroutine that is holding a read lease lock while reading
	// from a file.
	//
	// Lock ordering
	// -------------
	//
	// Define < to be the minimum strict partial order satisfying:
	//
	//  1. For any read/write lease W, W < leaser.
	//  2. For any read lease R, leaser < R.
	//
	// In other words: read/write before leaser before read, and never hold two
	// locks from the same category together.
	mu syncutil.InvariantMutex

	// The current estimated total size of outstanding read/write leases. This is
	// only an estimate because we can't synchronize its update with a call to
	// the wrapped file to e.g. write or truncate.
	readWriteOutstanding int64

	// All outstanding read leases, ordered by recency of use.
	//
	// INVARIANT: Each element is of type *readLease
	// INVARIANT: No element has been revoked.
	readLeases list.List

	// The sum of all outstanding read lease sizes.
	//
	// INVARIANT: Equal to the sum over readLeases sizes.
	// INVARIANT: 0 <= readOutstanding
	// INVARIANT: readOutstanding <= max(0, limit - readWriteOutstanding)
	readOutstanding int64

	// Index of read leases by pointer.
	//
	// INVARIANT: Is an index of exactly the elements of readLeases
	readLeasesIndex map[*readLease]*list.Element
}

// Create a new file leaser that uses the supplied directory for temporary
// files (before unlinking them) and attempts to keep usage in bytes below the
// given limit. If dir is empty, the system default will be used.
//
// Usage may exceed the given limit if there are read/write leases whose total
// size exceeds the limit, since such leases cannot be revoked.
func NewFileLeaser(
	dir string,
	limitBytes int64) (fl *FileLeaser) {
	fl = &FileLeaser{
		dir:             dir,
		limit:           limitBytes,
		readLeasesIndex: make(map[*readLease]*list.Element),
	}

	fl.mu = syncutil.NewInvariantMutex(fl.checkInvariants)

	return
}

// Create a new anonymous file, and return a read/write lease for it. The
// read/write lease will pin resources until rwl.Downgrade is called. It need
// not be called if the process is exiting.
func (fl *FileLeaser) NewFile() (rwl ReadWriteLease, err error) {
	// Create an anonymous file.
	f, err := fsutil.AnonymousFile(fl.dir)
	if err != nil {
		err = fmt.Errorf("AnonymousFile: %v", err)
		return
	}

	// Wrap a lease around it.
	rwl = newReadWriteLease(fl, 0, f)

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func maxInt64(a int64, b int64) int64 {
	if a > b {
		return a
	}

	return b
}

// LOCKS_REQUIRED(fl.mu)
func (fl *FileLeaser) checkInvariants() {
	// INVARIANT: Each element is of type *readLease
	// INVARIANT: No element has been revoked.
	for e := fl.readLeases.Front(); e != nil; e = e.Next() {
		rl := e.Value.(*readLease)
		rl.Mu.Lock()

		if rl.revoked() {
			panic("Found revoked read lease")
		}

		rl.Mu.Unlock()
	}

	// INVARIANT: Equal to the sum over readLeases sizes.
	var sum int64
	for e := fl.readLeases.Front(); e != nil; e = e.Next() {
		rl := e.Value.(*readLease)
		sum += rl.Size()
	}

	if fl.readOutstanding != sum {
		panic(fmt.Sprintf(
			"readOutstanding mismatch: %v vs. %v",
			fl.readOutstanding,
			sum))
	}

	// INVARIANT: 0 <= readOutstanding
	if !(0 <= fl.readOutstanding) {
		panic(fmt.Sprintf("Unexpected readOutstanding: %v", fl.readOutstanding))
	}

	// INVARIANT: readOutstanding <= max(0, limit - readWriteOutstanding)
	if !(fl.readOutstanding <= maxInt64(0, fl.limit-fl.readWriteOutstanding)) {
		panic(fmt.Sprintf(
			"Unexpected readOutstanding: %v. limit: %v, readWriteOutstanding: %v",
			fl.readOutstanding,
			fl.limit,
			fl.readWriteOutstanding))
	}

	// INVARIANT: Is an index of exactly the elements of readLeases
	if len(fl.readLeasesIndex) != fl.readLeases.Len() {
		panic(fmt.Sprintf(
			"readLeasesIndex length mismatch: %v vs. %v",
			len(fl.readLeasesIndex),
			fl.readLeases.Len()))
	}

	for e := fl.readLeases.Front(); e != nil; e = e.Next() {
		if fl.readLeasesIndex[e.Value.(*readLease)] != e {
			panic("Mismatch in readLeasesIndex")
		}
	}
}

// Add the supplied delta to the leaser's view of outstanding read/write lease
// bytes, then revoke read leases until we're under limit or we run out of
// leases to revoke.
//
// Called by readWriteLease while holding its lock.
//
// LOCKS_EXCLUDED(fl.mu)
func (fl *FileLeaser) addReadWriteByteDelta(delta int64) {
	fl.readWriteOutstanding += delta
	fl.evict()
}

// LOCKS_REQUIRED(fl.mu)
func (fl *FileLeaser) overLimit() bool {
	return fl.readOutstanding+fl.readWriteOutstanding > fl.limit
}

// Revoke read leases until we're under limit or we run out of things to revoke.
//
// LOCKS_REQUIRED(fl.mu)
func (fl *FileLeaser) evict() {
	for fl.overLimit() {
		// Do we have anything to revoke?
		lru := fl.readLeases.Back()
		if lru == nil {
			return
		}

		_ = lru.Value.(*readLease)
		panic("TODO")
	}
}

// Downgrade the supplied read/write lease, given its current size and the
// underlying file.
//
// Called by readWriteLease with its lock held.
//
// LOCKS_EXCLUDED(fl.mu)
func (fl *FileLeaser) downgrade(
	rwl *readWriteLease,
	size int64,
	file *os.File) (rl ReadLease) {
	// Create the read lease.
	rlTyped := newReadLease(size, fl, file)
	rl = rlTyped

	// Update the leaser's state, noting the new read lease and that the
	// read/write lease has gone away.
	fl.mu.Lock()
	defer fl.mu.Unlock()

	fl.readWriteOutstanding -= size
	fl.readOutstanding += size

	e := fl.readLeases.PushFront(rl)
	fl.readLeasesIndex[rlTyped] = e

	// Ensure that we're not now over capacity.
	fl.evict()

	return
}

// Upgrade the supplied read lease.
//
// Called by readLease with no lock held.
//
// LOCKS_EXCLUDED(fl.mu, rl.Mu)
func (fl *FileLeaser) upgrade(rl *readLease) (rwl ReadWriteLease) {
	// Grab each lock in turn.
	fl.mu.Lock()
	defer fl.mu.Unlock()

	rl.Mu.Lock()
	defer rl.Mu.Unlock()

	// Has the lease already been revoked?
	if rl.revoked() {
		return
	}

	size := rl.Size()

	// Update leaser state.
	fl.readWriteOutstanding += size
	fl.readOutstanding -= size

	e := fl.readLeasesIndex[rl]
	delete(fl.readLeasesIndex, rl)
	fl.readLeases.Remove(e)

	// Extract the interesting information from the read lease, leaving it an
	// empty husk.
	file := rl.release()

	// Create the read/write lease, telling it that we already know its initial
	// size.
	rwl = newReadWriteLease(fl, size, file)

	return
}

// Forcibly revoke the supplied read lease.
//
// LOCKS_REQUIRED(rl, fl.mu)
func (fl *FileLeaser) revoke(rl *readLease) {
	panic("TODO")
}

// Called by the read lease when the user wants to manually revoke it.
//
// LOCKS_EXCLUDED(fl.mu)
// LOCKS_EXCLUDED(rl.Mu)
func (fl *FileLeaser) revokeVoluntarily(rl *readLease) {
	// Grab each lock in turn.
	fl.mu.Lock()
	defer fl.mu.Unlock()

	rl.Mu.Lock()
	defer rl.Mu.Unlock()

	// Has the lease already been revoked?
	if rl.revoked() {
		return
	}

	panic("TODO")
}
<commit_msg>Don't unlock without defer. It confuses panic output.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lease

import (
	"container/list"
	"fmt"
	"os"

	"github.com/jacobsa/fuse/fsutil"
	"github.com/jacobsa/gcloud/syncutil"
)

// A type that manages read and read/write leases for anonymous temporary files.
//
// Safe for concurrent access. Must be created with NewFileLeaser.
type FileLeaser struct {
	/////////////////////////
	// Constant data
	/////////////////////////

	dir   string
	limit int64

	/////////////////////////
	// Mutable state
	/////////////////////////

	// A lock that guards the mutable state in this struct. Usually this is used
	// only for light weight operations, but while evicting it may require
	// waiting on a goroutine that is holding a read lease lock while reading
	// from a file.
	//
	// Lock ordering
	// -------------
	//
	// Define < to be the minimum strict partial order satisfying:
	//
	//  1. For any read/write lease W, W < leaser.
	//  2. For any read lease R, leaser < R.
	//
	// In other words: read/write before leaser before read, and never hold two
	// locks from the same category together.
	mu syncutil.InvariantMutex

	// The current estimated total size of outstanding read/write leases. This is
	// only an estimate because we can't synchronize its update with a call to
	// the wrapped file to e.g. write or truncate.
	readWriteOutstanding int64

	// All outstanding read leases, ordered by recency of use.
	//
	// INVARIANT: Each element is of type *readLease
	// INVARIANT: No element has been revoked.
	readLeases list.List

	// The sum of all outstanding read lease sizes.
	//
	// INVARIANT: Equal to the sum over readLeases sizes.
	// INVARIANT: 0 <= readOutstanding
	// INVARIANT: readOutstanding <= max(0, limit - readWriteOutstanding)
	readOutstanding int64

	// Index of read leases by pointer.
	//
	// INVARIANT: Is an index of exactly the elements of readLeases
	readLeasesIndex map[*readLease]*list.Element
}

// Create a new file leaser that uses the supplied directory for temporary
// files (before unlinking them) and attempts to keep usage in bytes below the
// given limit. If dir is empty, the system default will be used.
//
// Usage may exceed the given limit if there are read/write leases whose total
// size exceeds the limit, since such leases cannot be revoked.
func NewFileLeaser(
	dir string,
	limitBytes int64) (fl *FileLeaser) {
	fl = &FileLeaser{
		dir:             dir,
		limit:           limitBytes,
		readLeasesIndex: make(map[*readLease]*list.Element),
	}

	fl.mu = syncutil.NewInvariantMutex(fl.checkInvariants)

	return
}

// Create a new anonymous file, and return a read/write lease for it. The
// read/write lease will pin resources until rwl.Downgrade is called. It need
// not be called if the process is exiting.
func (fl *FileLeaser) NewFile() (rwl ReadWriteLease, err error) {
	// Create an anonymous file.
	f, err := fsutil.AnonymousFile(fl.dir)
	if err != nil {
		err = fmt.Errorf("AnonymousFile: %v", err)
		return
	}

	// Wrap a lease around it.
	rwl = newReadWriteLease(fl, 0, f)

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func maxInt64(a int64, b int64) int64 {
	if a > b {
		return a
	}

	return b
}

// LOCKS_REQUIRED(fl.mu)
func (fl *FileLeaser) checkInvariants() {
	// INVARIANT: Each element is of type *readLease
	// INVARIANT: No element has been revoked.
	for e := fl.readLeases.Front(); e != nil; e = e.Next() {
		rl := e.Value.(*readLease)
		func() {
			rl.Mu.Lock()
			defer rl.Mu.Unlock()

			if rl.revoked() {
				panic("Found revoked read lease")
			}
		}()
	}

	// INVARIANT: Equal to the sum over readLeases sizes.
	var sum int64
	for e := fl.readLeases.Front(); e != nil; e = e.Next() {
		rl := e.Value.(*readLease)
		sum += rl.Size()
	}

	if fl.readOutstanding != sum {
		panic(fmt.Sprintf(
			"readOutstanding mismatch: %v vs. %v",
			fl.readOutstanding,
			sum))
	}

	// INVARIANT: 0 <= readOutstanding
	if !(0 <= fl.readOutstanding) {
		panic(fmt.Sprintf("Unexpected readOutstanding: %v", fl.readOutstanding))
	}

	// INVARIANT: readOutstanding <= max(0, limit - readWriteOutstanding)
	if !(fl.readOutstanding <= maxInt64(0, fl.limit-fl.readWriteOutstanding)) {
		panic(fmt.Sprintf(
			"Unexpected readOutstanding: %v. limit: %v, readWriteOutstanding: %v",
			fl.readOutstanding,
			fl.limit,
			fl.readWriteOutstanding))
	}

	// INVARIANT: Is an index of exactly the elements of readLeases
	if len(fl.readLeasesIndex) != fl.readLeases.Len() {
		panic(fmt.Sprintf(
			"readLeasesIndex length mismatch: %v vs. %v",
			len(fl.readLeasesIndex),
			fl.readLeases.Len()))
	}

	for e := fl.readLeases.Front(); e != nil; e = e.Next() {
		if fl.readLeasesIndex[e.Value.(*readLease)] != e {
			panic("Mismatch in readLeasesIndex")
		}
	}
}

// Add the supplied delta to the leaser's view of outstanding read/write lease
// bytes, then revoke read leases until we're under limit or we run out of
// leases to revoke.
//
// Called by readWriteLease while holding its lock.
//
// LOCKS_EXCLUDED(fl.mu)
func (fl *FileLeaser) addReadWriteByteDelta(delta int64) {
	fl.readWriteOutstanding += delta
	fl.evict()
}

// LOCKS_REQUIRED(fl.mu)
func (fl *FileLeaser) overLimit() bool {
	return fl.readOutstanding+fl.readWriteOutstanding > fl.limit
}

// Revoke read leases until we're under limit or we run out of things to revoke.
//
// LOCKS_REQUIRED(fl.mu)
func (fl *FileLeaser) evict() {
	for fl.overLimit() {
		// Do we have anything to revoke?
		lru := fl.readLeases.Back()
		if lru == nil {
			return
		}

		// Revoke it.
		rl := lru.Value.(*readLease)
		func() {
			rl.Mu.Lock()
			defer rl.Mu.Unlock()

			fl.revoke(rl)
		}()
	}
}

// Downgrade the supplied read/write lease, given its current size and the
// underlying file.
//
// Called by readWriteLease with its lock held.
//
// LOCKS_EXCLUDED(fl.mu)
func (fl *FileLeaser) downgrade(
	rwl *readWriteLease,
	size int64,
	file *os.File) (rl ReadLease) {
	// Create the read lease.
	rlTyped := newReadLease(size, fl, file)
	rl = rlTyped

	// Update the leaser's state, noting the new read lease and that the
	// read/write lease has gone away.
	fl.mu.Lock()
	defer fl.mu.Unlock()

	fl.readWriteOutstanding -= size
	fl.readOutstanding += size

	e := fl.readLeases.PushFront(rl)
	fl.readLeasesIndex[rlTyped] = e

	// Ensure that we're not now over capacity.
	fl.evict()

	return
}

// Upgrade the supplied read lease.
//
// Called by readLease with no lock held.
//
// LOCKS_EXCLUDED(fl.mu, rl.Mu)
func (fl *FileLeaser) upgrade(rl *readLease) (rwl ReadWriteLease) {
	// Grab each lock in turn.
	fl.mu.Lock()
	defer fl.mu.Unlock()

	rl.Mu.Lock()
	defer rl.Mu.Unlock()

	// Has the lease already been revoked?
	if rl.revoked() {
		return
	}

	size := rl.Size()

	// Update leaser state.
	fl.readWriteOutstanding += size
	fl.readOutstanding -= size

	e := fl.readLeasesIndex[rl]
	delete(fl.readLeasesIndex, rl)
	fl.readLeases.Remove(e)

	// Extract the interesting information from the read lease, leaving it an
	// empty husk.
	file := rl.release()

	// Create the read/write lease, telling it that we already know its initial
	// size.
	rwl = newReadWriteLease(fl, size, file)

	return
}

// Forcibly revoke the supplied read lease.
//
// LOCKS_REQUIRED(fl.mu)
// LOCKS_REQUIRED(rl.Mu)
func (fl *FileLeaser) revoke(rl *readLease) {
	panic("TODO")
}

// Called by the read lease when the user wants to manually revoke it.
//
// LOCKS_EXCLUDED(fl.mu)
// LOCKS_EXCLUDED(rl.Mu)
func (fl *FileLeaser) revokeVoluntarily(rl *readLease) {
	// Grab each lock in turn.
	fl.mu.Lock()
	defer fl.mu.Unlock()

	rl.Mu.Lock()
	defer rl.Mu.Unlock()

	// Has the lease already been revoked?
	if rl.revoked() {
		return
	}

	panic("TODO")
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"math"
	"net"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/pborman/uuid"

	"flag"

	couchbasearray "github.com/andrewwebber/couchbase-array"
)

var servicePathFlag = flag.String("s", "/services/couchbase-array", "etcd directory")
var heartBeatFlag = flag.Int("h", 3, "heart beat loop in seconds")
var ttlFlag = flag.Int("ttl", 30, "time to live in seconds")
var debugFlag = flag.Bool("v", false, "verbose")
var rebalanceOnExitFlag = flag.Bool("r", false, "rebalance on exit")
var machineIdentiferFlag = flag.String("ip", "", "machine ip address")
var whatIfFlag = flag.Bool("t", false, "what if")
var cliBase = flag.String("cli", "/opt/couchbase/bin/couchbase-cli", "path to couchbase cli")
var masterNodeAnnouncePathFlag = flag.String("m", "/services/couchbase", "announce etcd path for the master IP")

func main() {
	log.SetFlags(log.Llongfile)
	flag.Parse()
	log.Println("Couchbase Cluster Node")
	couchbasearray.TTL = uint64(*ttlFlag)
	log.Printf("TTL %v\n", couchbasearray.TTL)

	machineIdentifier := *machineIdentiferFlag
	if machineIdentifier == "" {
		var err error
		machineIdentifier, err = getMachineIdentifier()
		if err != nil {
			log.Fatal(err)
		}
	}

	log.Printf("Machine ID: %s\n", machineIdentifier)

	sessionID := uuid.New()
	var isClusterMember bool

	go func() {
		for {
			announcments, err := couchbasearray.GetClusterAnnouncements(*servicePathFlag)
			if err != nil {
				log.Println(err)
				continue
			}

			machineState, ok := announcments[sessionID]
			if !ok {
				machineState = couchbasearray.NodeState{
					IPAddress:    machineIdentifier,
					SessionID:    sessionID,
					Master:       false,
					State:        "",
					DesiredState: ""}
			}

			currentStates, err := couchbasearray.GetClusterStates(*servicePathFlag)

			master, err := couchbasearray.GetMasterNode(currentStates)
			if err != nil {
				err := couchbasearray.AcquireLock(sessionID, *servicePathFlag+"/master", 5)
				if err == nil {
					stopScheduler := make(chan bool)
					go couchbasearray.StartScheduler(*servicePathFlag, *heartBeatFlag, stopScheduler, *masterNodeAnnouncePathFlag)
					go func() {
						failoverSet := false
						for {
							lockErr := couchbasearray.AcquireLock(sessionID, *servicePathFlag+"/master", 5)
							if lockErr != nil {
								log.Println(lockErr)
								stopScheduler <- true
								return
							}

							if !failoverSet {
								if failOverErr := setAutoFailover(machineIdentifier, 31); err != nil {
									log.Println(failOverErr)
								} else {
									failoverSet = true
								}
							}

							time.Sleep(4 * time.Second)
						}
					}()
				}

				if err != nil && err != couchbasearray.ErrLockInUse {
					log.Println(err)
					continue
				}
			} else {
				if state, ok := currentStates[sessionID]; ok {
					if state.DesiredState != machineState.State {
						log.Printf("DesiredState: %s - Current State: %s", state.DesiredState, machineState.State)

						switch state.DesiredState {
						case couchbasearray.SchedulerStateClustered:
							log.Println("rebalancing")

							if master.IPAddress == machineIdentifier {
								log.Println("Already master no action required")
							} else {
								log.Printf("rebalancing with master node %s\n", master.IPAddress)
								if !*whatIfFlag {
									if isClusterMember {
										err = recoverNode(master.IPAddress, machineIdentifier)
									}

									err = rebalanceNode(master.IPAddress, machineIdentifier)
								}
							}

							if err == nil {
								machineState.State = state.DesiredState
							} else {
								log.Println(err)
							}

						case couchbasearray.SchedulerStateNew:
							log.Println("adding server to cluster")
							master, err := couchbasearray.GetMasterNode(currentStates)
							if err != nil {
								log.Println(err)
							} else {
								if master.IPAddress == machineIdentifier {
									log.Println("Already master no action required")
								} else {
									log.Printf("Adding to master node %s\n", master.IPAddress)
									if !*whatIfFlag {
										isClusterMember, err = addNodeToCluster(master.IPAddress, machineIdentifier)
									}
								}

								if err == nil {
									machineState.State = state.DesiredState
								} else {
									log.Println(err)
								}
							}
						default:
							log.Println(state.DesiredState)
							log.Fatal("unknown state")
						}
					}
				} else {
					log.Println("Running")
				}
			}

			err = couchbasearray.SetClusterAnnouncement(*servicePathFlag, machineState)
			if err != nil {
				log.Println(err)
			}

			time.Sleep(time.Duration(*heartBeatFlag) * time.Second)
		}
	}()

	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGKILL)
	log.Println(<-ch)
	log.Println("Failing over")

	currentStates, err := couchbasearray.GetClusterStates(*servicePathFlag)
	if err != nil {
		log.Fatal(err)
		return
	}

	master, err := couchbasearray.GetMasterNode(currentStates)
	if err != nil {
		log.Fatal(err)
		return
	}

	err = failoverClusterNode(master.IPAddress, machineIdentifier)
	if err != nil {
		log.Fatal(err)
	}

	if *rebalanceOnExitFlag {
		err = rebalanceNode(master.IPAddress, machineIdentifier)
		if err != nil {
			log.Fatal(err)
		}
	}
}

func getMachineIdentifier() (string, error) {

	addrs, err := net.InterfaceAddrs()
	if err != nil {
		log.Fatalln(err)
	}

	var result string
	for _, a := range addrs {
		if ipnet, ok := a.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
			if ipnet.IP.To4() != nil {
				result = ipnet.IP.String()
				log.Println(ipnet.Network())
				log.Printf("Found IP %s\n", result)
				break
			}
		}
	}

	if result != "" {
		return result, nil
	}

	return os.Hostname()
}

type function func() error

func exponential(operation function, maxRetries int) error {
	var err error
	var sleepTime int
	for i := 0; i < maxRetries; i++ {
		err = operation()
		if err == nil {
			return nil
		}
		if i == 0 {
			sleepTime = 1
		} else {
			sleepTime = int(math.Exp2(float64(i)) * 100)
		}
		time.Sleep(time.Duration(sleepTime) * time.Millisecond)
		log.Printf("Retry exponential: Attempt %d, sleep %d", i, sleepTime)
	}

	return err
}
<commit_msg>stateful sets<commit_after>package main

import (
	"log"
	"math"
	"net"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/pborman/uuid"

	"flag"

	couchbasearray "github.com/andrewwebber/couchbase-array"
)

var servicePathFlag = flag.String("s", "/services/couchbase-array", "etcd directory")
var heartBeatFlag = flag.Int("h", 3, "heart beat loop in seconds")
var ttlFlag = flag.Int("ttl", 30, "time to live in seconds")
var debugFlag = flag.Bool("v", false, "verbose")
var rebalanceOnExitFlag = flag.Bool("r", false, "rebalance on exit")
var machineIdentiferFlag = flag.String("ip", "", "machine ip address")
var whatIfFlag = flag.Bool("t", false, "what if")
var cliBase = flag.String("cli", "/opt/couchbase/bin/couchbase-cli", "path to couchbase cli")
var statefulSet = flag.Bool("statefulset", false, "use hostnames instead of ip addresses")
var masterNodeAnnouncePathFlag = flag.String("m", "/services/couchbase", "announce etcd path for the master IP")

func main() {
	log.SetFlags(log.Llongfile)
	flag.Parse()
	log.Println("Couchbase Cluster Node")
	couchbasearray.TTL = uint64(*ttlFlag)
	log.Printf("TTL %v\n", couchbasearray.TTL)

	machineIdentifier := *machineIdentiferFlag
	if machineIdentifier == "" {
		var err error
		machineIdentifier, err = getMachineIdentifier()
		if err != nil {
			log.Fatal(err)
		}
	}

	log.Printf("Machine ID: %s\n", machineIdentifier)

	sessionID := uuid.New()
	var isClusterMember bool

	go func() {
		for {
			announcments, err := couchbasearray.GetClusterAnnouncements(*servicePathFlag)
			if err != nil {
				log.Println(err)
				continue
			}

			machineState, ok := announcments[sessionID]
			if !ok {
				machineState = couchbasearray.NodeState{
					IPAddress:    machineIdentifier,
					SessionID:    sessionID,
					Master:       false,
					State:        "",
					DesiredState: ""}
			}

			currentStates, err := couchbasearray.GetClusterStates(*servicePathFlag)

			master, err := couchbasearray.GetMasterNode(currentStates)
			if err != nil {
				err := couchbasearray.AcquireLock(sessionID, *servicePathFlag+"/master", 5)
				if err == nil {
					stopScheduler := make(chan bool)
					go couchbasearray.StartScheduler(*servicePathFlag, *heartBeatFlag, stopScheduler, *masterNodeAnnouncePathFlag)
					go func() {
						failoverSet := false
						for {
							lockErr := couchbasearray.AcquireLock(sessionID, *servicePathFlag+"/master", 5)
							if lockErr != nil {
								log.Println(lockErr)
								stopScheduler <- true
								return
							}

							if !failoverSet {
								if failOverErr := setAutoFailover(machineIdentifier, 31); err != nil {
									log.Println(failOverErr)
								} else {
									failoverSet = true
								}
							}

							time.Sleep(4 * time.Second)
						}
					}()
				}

				if err != nil && err != couchbasearray.ErrLockInUse {
					log.Println(err)
					continue
				}
			} else {
				if state, ok := currentStates[sessionID]; ok {
					if state.DesiredState != machineState.State {
						log.Printf("DesiredState: %s - Current State: %s", state.DesiredState, machineState.State)

						switch state.DesiredState {
						case couchbasearray.SchedulerStateClustered:
							log.Println("rebalancing")

							if master.IPAddress == machineIdentifier {
								log.Println("Already master no action required")
							} else {
								log.Printf("rebalancing with master node %s\n", master.IPAddress)
								if !*whatIfFlag {
									if isClusterMember {
										err = recoverNode(master.IPAddress, machineIdentifier)
									}

									err = rebalanceNode(master.IPAddress, machineIdentifier)
								}
							}

							if err == nil {
								machineState.State = state.DesiredState
							} else {
								log.Println(err)
							}

						case couchbasearray.SchedulerStateNew:
							log.Println("adding server to cluster")
							master, err := couchbasearray.GetMasterNode(currentStates)
							if err != nil {
								log.Println(err)
							} else {
								if master.IPAddress == machineIdentifier {
									log.Println("Already master no action required")
								} else {
									log.Printf("Adding to master node %s\n", master.IPAddress)
									if !*whatIfFlag {
										isClusterMember, err = addNodeToCluster(master.IPAddress, machineIdentifier)
									}
								}

								if err == nil {
									machineState.State = state.DesiredState
								} else {
									log.Println(err)
								}
							}
						default:
							log.Println(state.DesiredState)
							log.Fatal("unknown state")
						}
					}
				} else {
					log.Println("Running")
				}
			}

			err = couchbasearray.SetClusterAnnouncement(*servicePathFlag, machineState)
			if err != nil {
				log.Println(err)
			}

			time.Sleep(time.Duration(*heartBeatFlag) * time.Second)
		}
	}()

	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGKILL)
	log.Println(<-ch)
	log.Println("Failing over")

	currentStates, err := couchbasearray.GetClusterStates(*servicePathFlag)
	if err != nil {
		log.Fatal(err)
		return
	}

	master, err := couchbasearray.GetMasterNode(currentStates)
	if err != nil {
		log.Fatal(err)
		return
	}

	err = failoverClusterNode(master.IPAddress, machineIdentifier)
	if err != nil {
		log.Fatal(err)
	}

	if *rebalanceOnExitFlag {
		err = rebalanceNode(master.IPAddress, machineIdentifier)
		if err != nil {
			log.Fatal(err)
		}
	}
}

func getMachineIdentifier() (string, error) {
	if *statefulSet {
		return os.Hostname()
	}

	addrs, err := net.InterfaceAddrs()
	if err != nil {
		log.Fatalln(err)
	}

	var result string
	for _, a := range addrs {
		if ipnet, ok := a.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
			if ipnet.IP.To4() != nil {
				result = ipnet.IP.String()
				log.Println(ipnet.Network())
				log.Printf("Found IP %s\n", result)
				break
			}
		}
	}

	if result != "" {
		return result, nil
	}

	return os.Hostname()
}

type function func() error

func exponential(operation function, maxRetries int) error {
	var err error
	var sleepTime int
	for i := 0; i < maxRetries; i++ {
		err = operation()
		if err == nil {
			return nil
		}
		if i == 0 {
			sleepTime = 1
		} else {
			sleepTime = int(math.Exp2(float64(i)) * 100)
		}
		time.Sleep(time.Duration(sleepTime) * time.Millisecond)
		log.Printf("Retry exponential: Attempt %d, sleep %d", i, sleepTime)
	}

	return err
}
<|endoftext|>


<commit_before>package s3api

import (
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, prefix string, lastTsNs int64) error {

	processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error {

		message := resp.EventNotification
		if message.NewEntry == nil {
			return nil
		}

		dir := resp.Directory

		if message.NewParentPath != "" {
			dir = message.NewParentPath
		}
		if dir == filer.IamConfigDirecotry && message.NewEntry.Name == filer.IamIdentityFile {
			err := util.Retry("updateIamIdentity", func() error {
				return s3a.iam.loadS3ApiConfigurationFromBytes(message.NewEntry.Content)
			})
			if err != nil {
				return err
			}
			glog.V(0).Infof("updated %s/%s", filer.IamConfigDirecotry, filer.IamIdentityFile)
		}

		return nil
	}

	return util.Retry("followIamChanges", func() error {
		return pb.WithFilerClientFollowMetadata(s3a, clientName, prefix, lastTsNs, 0, processEventFn, true)
	})

}
<commit_msg>Retry save and update IAM identity https://github.com/chrislusf/seaweedfs/issues/2242<commit_after>package s3api

import (
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, prefix string, lastTsNs int64) error {

	processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error {

		message := resp.EventNotification
		if message.NewEntry == nil {
			return nil
		}

		dir := resp.Directory

		if message.NewParentPath != "" {
			dir = message.NewParentPath
		}
		if dir == filer.IamConfigDirecotry && message.NewEntry.Name == filer.IamIdentityFile {
			if err := s3a.iam.loadS3ApiConfigurationFromBytes(message.NewEntry.Content); err != nil {
				return err
			}
			glog.V(0).Infof("updated %s/%s", filer.IamConfigDirecotry, filer.IamIdentityFile)
		}

		return nil
	}

	return util.Retry("followIamChanges", func() error {
		return pb.WithFilerClientFollowMetadata(s3a, clientName, prefix, lastTsNs, 0, processEventFn, true)
	})

}
<|endoftext|>


<commit_before>package s3api

import (
	"crypto/md5"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/util"
)

func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {

	dstBucket, dstObject := getBucketAndObject(r)

	// Copy source path.
	cpSrcPath, err := url.QueryUnescape(r.Header.Get("X-Amz-Copy-Source"))
	if err != nil {
		// Save unescaped string as is.
		cpSrcPath = r.Header.Get("X-Amz-Copy-Source")
	}

	srcBucket, srcObject := pathToBucketAndObject(cpSrcPath)
	// If source object is empty or bucket is empty, reply back invalid copy source.
	if srcObject == "" || srcBucket == "" {
		writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
		return
	}

	if srcBucket == dstBucket && srcObject == dstObject {
		writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
		return
	}

	dstUrl := fmt.Sprintf("http://%s%s/%s%s?collection=%s",
		s3a.option.Filer, s3a.option.BucketsPath, dstBucket, dstObject, dstBucket)
	srcUrl := fmt.Sprintf("http://%s%s/%s%s",
		s3a.option.Filer, s3a.option.BucketsPath, srcBucket, srcObject)

	_, _, resp, err := util.DownloadFile(srcUrl)
	if err != nil {
		writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
		return
	}
	defer util.CloseResponse(resp)

	etag, errCode := s3a.putToFiler(r, dstUrl, resp.Body)

	if errCode != s3err.ErrNone {
		writeErrorResponse(w, errCode, r.URL)
		return
	}

	setEtag(w, etag)

	response := CopyObjectResult{
		ETag:         etag,
		LastModified: time.Now().UTC(),
	}

	writeSuccessResponseXML(w, encodeResponse(response))

}

func pathToBucketAndObject(path string) (bucket, object string) {
	path = strings.TrimPrefix(path, "/")
	parts := strings.SplitN(path, "/", 2)
	if len(parts) == 2 {
		return parts[0], "/" + parts[1]
	}
	return parts[0], "/"
}

type CopyPartResult struct {
	LastModified time.Time `xml:"LastModified"`
	ETag         string    `xml:"ETag"`
}

func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Request) {
	// https://docs.aws.amazon.com/AmazonS3/latest/dev/CopyingObjctsUsingRESTMPUapi.html
	// https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html
	dstBucket, dstObject := getBucketAndObject(r)

	// Copy source path.
	cpSrcPath, err := url.QueryUnescape(r.Header.Get("X-Amz-Copy-Source"))
	if err != nil {
		// Save unescaped string as is.
		cpSrcPath = r.Header.Get("X-Amz-Copy-Source")
	}

	srcBucket, srcObject := pathToBucketAndObject(cpSrcPath)
	// If source object is empty or bucket is empty, reply back invalid copy source.
	if srcObject == "" || srcBucket == "" {
		writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
		return
	}

	uploadID := r.URL.Query().Get("uploadId")
	partIDString := r.URL.Query().Get("partNumber")

	partID, err := strconv.Atoi(partIDString)
	if err != nil {
		writeErrorResponse(w, s3err.ErrInvalidPart, r.URL)
		return
	}

	// check partID with maximum part ID for multipart objects
	if partID > globalMaxPartID {
		writeErrorResponse(w, s3err.ErrInvalidMaxParts, r.URL)
		return
	}

	rangeHeader := r.Header.Get("x-amz-copy-source-range")

	dstUrl := fmt.Sprintf("http://%s%s/%s/%04d.part?collection=%s",
		s3a.option.Filer, s3a.genUploadsFolder(dstBucket), uploadID, partID, dstBucket)
	srcUrl := fmt.Sprintf("http://%s%s/%s%s",
		s3a.option.Filer, s3a.option.BucketsPath, srcBucket, srcObject)

	dataReader, err := util.ReadUrlAsReaderCloser(srcUrl, rangeHeader)
	if err != nil {
		writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
		return
	}
	defer dataReader.Close()

	var etag string
	if strings.HasSuffix(srcObject, "/") {
		if err := s3a.mkdir(s3a.option.BucketsPath, dstBucket+dstObject, nil); err != nil {
			writeErrorResponse(w, s3err.ErrInternalError, r.URL)
			return
		}
		etag = fmt.Sprintf("%x", md5.New().Sum(nil))
	} else {
		_etag, errCode := s3a.putToFiler(r, dstUrl, dataReader)
		etag = _etag
		if errCode != s3err.ErrNone {
			writeErrorResponse(w, errCode, r.URL)
			return
		}
	}

	setEtag(w, etag)

	response := CopyPartResult{
		ETag:         etag,
		LastModified: time.Now().UTC(),
	}

	writeSuccessResponseXML(w, encodeResponse(response))

}
<commit_msg>rollback<commit_after>package s3api

import (
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/util"
)

func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {

	dstBucket, dstObject := getBucketAndObject(r)

	// Copy source path.
	cpSrcPath, err := url.QueryUnescape(r.Header.Get("X-Amz-Copy-Source"))
	if err != nil {
		// Save unescaped string as is.
		cpSrcPath = r.Header.Get("X-Amz-Copy-Source")
	}

	srcBucket, srcObject := pathToBucketAndObject(cpSrcPath)
	// If source object is empty or bucket is empty, reply back invalid copy source.
	if srcObject == "" || srcBucket == "" {
		writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
		return
	}

	if srcBucket == dstBucket && srcObject == dstObject {
		writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
		return
	}

	dstUrl := fmt.Sprintf("http://%s%s/%s%s?collection=%s",
		s3a.option.Filer, s3a.option.BucketsPath, dstBucket, dstObject, dstBucket)
	srcUrl := fmt.Sprintf("http://%s%s/%s%s",
		s3a.option.Filer, s3a.option.BucketsPath, srcBucket, srcObject)

	_, _, resp, err := util.DownloadFile(srcUrl)
	if err != nil {
		writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
		return
	}
	defer util.CloseResponse(resp)

	etag, errCode := s3a.putToFiler(r, dstUrl, resp.Body)

	if errCode != s3err.ErrNone {
		writeErrorResponse(w, errCode, r.URL)
		return
	}

	setEtag(w, etag)

	response := CopyObjectResult{
		ETag:         etag,
		LastModified: time.Now().UTC(),
	}

	writeSuccessResponseXML(w, encodeResponse(response))

}

func pathToBucketAndObject(path string) (bucket, object string) {
	path = strings.TrimPrefix(path, "/")
	parts := strings.SplitN(path, "/", 2)
	if len(parts) == 2 {
		return parts[0], "/" + parts[1]
	}
	return parts[0], "/"
}

type CopyPartResult struct {
	LastModified time.Time `xml:"LastModified"`
	ETag         string    `xml:"ETag"`
}

func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Request) {
	// https://docs.aws.amazon.com/AmazonS3/latest/dev/CopyingObjctsUsingRESTMPUapi.html
	// https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html
	dstBucket, _ := getBucketAndObject(r)

	// Copy source path.
	cpSrcPath, err := url.QueryUnescape(r.Header.Get("X-Amz-Copy-Source"))
	if err != nil {
		// Save unescaped string as is.
		cpSrcPath = r.Header.Get("X-Amz-Copy-Source")
	}

	srcBucket, srcObject := pathToBucketAndObject(cpSrcPath)
	// If source object is empty or bucket is empty, reply back invalid copy source.
	if srcObject == "" || srcBucket == "" {
		writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
		return
	}

	uploadID := r.URL.Query().Get("uploadId")
	partIDString := r.URL.Query().Get("partNumber")

	partID, err := strconv.Atoi(partIDString)
	if err != nil {
		writeErrorResponse(w, s3err.ErrInvalidPart, r.URL)
		return
	}

	// check partID with maximum part ID for multipart objects
	if partID > globalMaxPartID {
		writeErrorResponse(w, s3err.ErrInvalidMaxParts, r.URL)
		return
	}

	rangeHeader := r.Header.Get("x-amz-copy-source-range")

	dstUrl := fmt.Sprintf("http://%s%s/%s/%04d.part?collection=%s",
		s3a.option.Filer, s3a.genUploadsFolder(dstBucket), uploadID, partID, dstBucket)
	srcUrl := fmt.Sprintf("http://%s%s/%s%s",
		s3a.option.Filer, s3a.option.BucketsPath, srcBucket, srcObject)

	dataReader, err := util.ReadUrlAsReaderCloser(srcUrl, rangeHeader)
	if err != nil {
		writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
		return
	}
	defer dataReader.Close()

	etag, errCode := s3a.putToFiler(r, dstUrl, dataReader)

	if errCode != s3err.ErrNone {
		writeErrorResponse(w, errCode, r.URL)
		return
	}

	setEtag(w, etag)

	response := CopyPartResult{
		ETag:         etag,
		LastModified: time.Now().UTC(),
	}

	writeSuccessResponseXML(w, encodeResponse(response))

}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"os"
	"os/signal"
	"runtime"
	"syscall"

	"github.com/ehalpern/go-mysql-elasticsearch/river"
	"github.com/juju/errors"
)

var configFile = flag.String("config", "./etc/river.toml", "go-mysql-elasticsearch config file")

var my_addr = flag.String("my_addr", "", "MySQL addr")
var my_user = flag.String("my_user", "", "MySQL user")
var my_pass = flag.String("my_pass", "", "MySQL password")
var es_addr = flag.String("es_addr", "", "Elasticsearch addr")
var data_dir = flag.String("data_dir", "", "path for go-mysql-elasticsearch to save data")
var server_id = flag.Int("server_id", 0, "MySQL server id, as a pseudo slave")
var flavor = flag.String("flavor", "", "flavor: mysql or mariadb")
var execution = flag.String("exec", "", "mysqldump execution path")
var max_actions = flag.Int("max_actions", 500, "number of actions to include in an elasticsearch bulk update")

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	flag.Parse()

	sc := make(chan os.Signal, 1)
	signal.Notify(sc,
		os.Kill,
		os.Interrupt,
		syscall.SIGHUP,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)

	cfg, err := river.NewConfigWithFile(*configFile)
	if err != nil {
		println(errors.ErrorStack(err))
		return
	}

	if len(*my_addr) > 0 {
		cfg.MyAddr = *my_addr
	}

	if len(*my_user) > 0 {
		cfg.MyUser = *my_user
	}

	if len(*my_pass) > 0 {
		cfg.MyPassword = *my_pass
	}

	if *server_id > 0 {
		cfg.ServerID = uint32(*server_id)
	}

	if len(*es_addr) > 0 {
		cfg.ESAddr = *es_addr
	}

	if len(*data_dir) > 0 {
		cfg.DataDir = *data_dir
	}

	if len(*flavor) > 0 {
		cfg.Flavor = *flavor
	}

	if len(*execution) > 0 {
		cfg.DumpExec = *execution
	}

	if *max_actions > 0 {
		cfg.MaxBulkActions = *max_actions
	}

	r, err := river.NewRiver(cfg)
	if err != nil {
		println(errors.ErrorStack(err))
		return
	}

	if err = r.Run(); err != nil {
		println(errors.ErrorStack(err))
	}

	<-sc
	r.Close()
}
<commit_msg>Fix problem with max_actions flag masking settings<commit_after>package main

import (
	"flag"
	"os"
	"os/signal"
	"runtime"
	"syscall"

	"github.com/ehalpern/go-mysql-elasticsearch/river"
	"github.com/juju/errors"
)

var configFile = flag.String("config", "./etc/river.toml", "go-mysql-elasticsearch config file")

var my_addr = flag.String("my_addr", "", "MySQL addr")
var my_user = flag.String("my_user", "", "MySQL user")
var my_pass = flag.String("my_pass", "", "MySQL password")
var es_addr = flag.String("es_addr", "", "Elasticsearch addr")
var data_dir = flag.String("data_dir", "", "path for go-mysql-elasticsearch to save data")
var server_id = flag.Int("server_id", 0, "MySQL server id, as a pseudo slave")
var flavor = flag.String("flavor", "", "flavor: mysql or mariadb")
var execution = flag.String("exec", "", "mysqldump execution path")
var max_actions = flag.Int("max_actions", 0, "number of actions to include in an elasticsearch bulk update")

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	flag.Parse()

	sc := make(chan os.Signal, 1)
	signal.Notify(sc,
		os.Kill,
		os.Interrupt,
		syscall.SIGHUP,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)

	cfg, err := river.NewConfigWithFile(*configFile)
	if err != nil {
		println(errors.ErrorStack(err))
		return
	}

	if len(*my_addr) > 0 {
		cfg.MyAddr = *my_addr
	}

	if len(*my_user) > 0 {
		cfg.MyUser = *my_user
	}

	if len(*my_pass) > 0 {
		cfg.MyPassword = *my_pass
	}

	if *server_id > 0 {
		cfg.ServerID = uint32(*server_id)
	}

	if len(*es_addr) > 0 {
		cfg.ESAddr = *es_addr
	}

	if len(*data_dir) > 0 {
		cfg.DataDir = *data_dir
	}

	if len(*flavor) > 0 {
		cfg.Flavor = *flavor
	}

	if len(*execution) > 0 {
		cfg.DumpExec = *execution
	}

	if *max_actions > 0 {
		cfg.MaxBulkActions = *max_actions
	}

	r, err := river.NewRiver(cfg)
	if err != nil {
		println(errors.ErrorStack(err))
		return
	}

	if err = r.Run(); err != nil {
		println(errors.ErrorStack(err))
	}

	<-sc
	r.Close()
}
<|endoftext|>