<commit_before>package gardenhealth_test

import (
	"errors"
	"os"
	"time"

	"code.cloudfoundry.org/executor/gardenhealth"
	"github.com/cloudfoundry/dropsonde/metric_sender/fake"
	"github.com/cloudfoundry/dropsonde/metrics"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"

	"code.cloudfoundry.org/clock/fakeclock"
	fakeexecutor "code.cloudfoundry.org/executor/fakes"
	"code.cloudfoundry.org/executor/gardenhealth/fakegardenhealth"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/lagertest"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
)

var _ = Describe("Runner", func() {
	var (
		runner                          *gardenhealth.Runner
		process                         ifrit.Process
		logger                          *lagertest.TestLogger
		checker                         *fakegardenhealth.FakeChecker
		executorClient                  *fakeexecutor.FakeClient
		sender                          *fake.FakeMetricSender
		fakeClock                       *fakeclock.FakeClock
		checkInterval, emissionInterval time.Duration
		timeoutDuration                 time.Duration
	)

	BeforeEach(func() {
		logger = lagertest.NewTestLogger("test")
		checker = &fakegardenhealth.FakeChecker{}
		executorClient = &fakeexecutor.FakeClient{}
		fakeClock = fakeclock.NewFakeClock(time.Now())
		checkInterval = 2 * time.Minute
		timeoutDuration = 1 * time.Minute
		emissionInterval = 30 * time.Second

		sender = fake.NewFakeMetricSender()
		metrics.Initialize(sender, nil)
	})

	JustBeforeEach(func() {
		runner = gardenhealth.NewRunner(checkInterval, emissionInterval, timeoutDuration, logger, checker, executorClient, fakeClock)
		process = ifrit.Background(runner)
	})

	AfterEach(func() {
		ginkgomon.Interrupt(process)
	})

	Describe("Run", func() {
		Context("When garden is immediately unhealthy", func() {
			Context("because the health check fails", func() {
				var checkErr = gardenhealth.UnrecoverableError("nope")
				BeforeEach(func() {
					checker.HealthcheckReturns(checkErr)
					executorClient.HealthyReturns(false)
				})

				It("fails without becoming ready", func() {
					Eventually(process.Wait()).Should(Receive(Equal(checkErr)))
					Consistently(process.Ready()).ShouldNot(BeClosed())
				})

				It("emits a metric for unhealthy cell", func() {
					Eventually(process.Wait()).Should(Receive(Equal(checkErr)))
					Eventually(func() float64 {
						return sender.GetValue("UnhealthyCell").Value
					}).Should(Equal(float64(1)))
				})
			})

			Context("because the health check timed out", func() {
				var blockHealthcheck chan struct{}

				BeforeEach(func() {
					blockHealthcheck = make(chan struct{})
					checker.HealthcheckStub = func(lager.Logger) error {
						<-blockHealthcheck
						return nil
					}
				})

				JustBeforeEach(func() {
					fakeClock.WaitForWatcherAndIncrement(timeoutDuration)
				})

				AfterEach(func() {
					// Send to the channel to eliminate the race
					blockHealthcheck <- struct{}{}
					close(blockHealthcheck)
					blockHealthcheck = nil
				})

				It("fails without becoming ready", func() {
					Eventually(process.Wait()).Should(Receive(Equal(gardenhealth.HealthcheckTimeoutError{})))
					Consistently(process.Ready()).ShouldNot(BeClosed())
				})

				It("emits a metric for unhealthy cell", func() {
					Eventually(process.Wait()).Should(Receive(Equal(gardenhealth.HealthcheckTimeoutError{})))
					Eventually(func() float64 {
						return sender.GetValue("UnhealthyCell").Value
					}).Should(Equal(float64(1)))
				})
			})
		})

		Context("When garden is healthy", func() {
			It("sets healthy to true only once", func() {
				Eventually(executorClient.SetHealthyCallCount).Should(Equal(1))
				_, healthy := executorClient.SetHealthyArgsForCall(0)
				Expect(healthy).Should(Equal(true))
				Expect(executorClient.SetHealthyCallCount()).To(Equal(1))
			})

			It("continues to check at the correct interval", func() {
				Eventually(checker.HealthcheckCallCount).Should(Equal(1))
				Eventually(process.Ready()).Should(BeClosed())

				fakeClock.WaitForWatcherAndIncrement(checkInterval)
				Eventually(checker.HealthcheckCallCount).Should(Equal(2))
				Eventually(logger).Should(gbytes.Say("check-complete"))

				fakeClock.WaitForWatcherAndIncrement(checkInterval)
				Eventually(checker.HealthcheckCallCount).Should(Equal(3))
				Eventually(logger).Should(gbytes.Say("check-complete"))

				fakeClock.WaitForWatcherAndIncrement(checkInterval)
				Eventually(checker.HealthcheckCallCount).Should(Equal(4))
				Eventually(logger).Should(gbytes.Say("check-complete"))
			})

			It("emits a metric for healthy cell", func() {
				Eventually(func () float64 {
					return sender.GetValue("UnhealthyCell").Value
				}).Should(Equal(float64(0)))
			})
		})

		Context("when garden is intermittently healthy", func() {
			var checkErr = errors.New("nope")

			BeforeEach(func() {
				executorClient.HealthyReturns(true)
			})

			It("Sets healthy to false after it fails, then to true after success and emits respective metrics", func() {
				Eventually(executorClient.SetHealthyCallCount).Should(Equal(1))
				_, healthy := executorClient.SetHealthyArgsForCall(0)
				Expect(healthy).Should(Equal(true))
				Expect(sender.GetValue("UnhealthyCell").Value).To(Equal(float64(0)))

				checker.HealthcheckReturns(checkErr)
				executorClient.HealthyReturns(false)
				fakeClock.WaitForWatcherAndIncrement(checkInterval)

				Eventually(executorClient.SetHealthyCallCount).Should(Equal(2))
				_, healthy = executorClient.SetHealthyArgsForCall(1)
				Expect(healthy).Should(Equal(false))
				Expect(sender.GetValue("UnhealthyCell").Value).To(Equal(float64(1)))

				checker.HealthcheckReturns(nil)
				executorClient.HealthyReturns(true)
				fakeClock.WaitForWatcherAndIncrement(checkInterval)

				Eventually(executorClient.SetHealthyCallCount).Should(Equal(3))
				_, healthy = executorClient.SetHealthyArgsForCall(2)
				Expect(healthy).Should(Equal(true))
				Expect(sender.GetValue("UnhealthyCell").Value).To(Equal(float64(0)))
			})
		})

		Context("When the healthcheck times out", func() {
			var blockHealthcheck chan struct{}

			BeforeEach(func() {
				blockHealthcheck = make(chan struct{})
				checker.HealthcheckStub = func(lager.Logger) error {
					logger.Info("blocking")
					<-blockHealthcheck
					logger.Info("unblocking")
					return nil
				}
			})

			AfterEach(func() {
				close(blockHealthcheck)
			})

			It("sets the executor to unhealthy and emits the unhealthy metric", func() {
				Eventually(blockHealthcheck).Should(BeSent(struct{}{}))
				Eventually(executorClient.SetHealthyCallCount).Should(Equal(1))

				fakeClock.WaitForWatcherAndIncrement(checkInterval)
				Eventually(checker.HealthcheckCallCount).Should(Equal(2))

				fakeClock.WaitForWatcherAndIncrement(timeoutDuration)

				Eventually(executorClient.SetHealthyCallCount).Should(Equal(2))
				_, healthy := executorClient.SetHealthyArgsForCall(1)
				Expect(healthy).Should(Equal(false))
				Eventually(func() float64 {
					return sender.GetValue("UnhealthyCell").Value
				}).Should(Equal(float64(1)))
			})
		})

		Context("When the runner is signaled", func() {
			Context("during the initial health check", func() {
				var blockHealthcheck chan struct{}

				BeforeEach(func() {
					blockHealthcheck = make(chan struct{})
					checker.HealthcheckStub = func(lager.Logger) error {
						<-blockHealthcheck
						return nil
					}
				})

				JustBeforeEach(func() {
					process.Signal(os.Interrupt)
				})

				It("exits with no error", func() {
					Eventually(process.Wait()).Should(Receive(BeNil()))
				})
			})

			Context("After the initial health check", func() {
				It("exits imediately with no error", func() {
					Eventually(executorClient.SetHealthyCallCount).Should(Equal(1))

					process.Signal(os.Interrupt)
					Eventually(process.Wait()).Should(Receive(BeNil()))
				})
			})
		})

		Context("UnhealthyCell metric emission", func() {
			It("emits the UnhealthyCell every emitInterval", func() {
				Eventually(executorClient.HealthyCallCount).Should(Equal(1))
				Eventually(func() float64 {
					return sender.GetValue("UnhealthyCell").Value
				}).Should(Equal(float64(1)))

				executorClient.HealthyReturns(true)
				fakeClock.WaitForWatcherAndIncrement(emissionInterval)

				Eventually(executorClient.HealthyCallCount).Should(Equal(2))
				Eventually(func() float64 {
					return sender.GetValue("UnhealthyCell").Value
				}).Should(Equal(float64(0)))

				executorClient.HealthyReturns(false)
				fakeClock.WaitForWatcherAndIncrement(emissionInterval)

				Eventually(executorClient.HealthyCallCount).Should(Equal(3))
				Eventually(func() float64 {
					return sender.GetValue("UnhealthyCell").Value
				}).Should(Equal(float64(1)))
			})
		})
	})
})
<commit_msg>Fix flaky test<commit_after>package gardenhealth_test

import (
	"errors"
	"os"
	"time"

	"code.cloudfoundry.org/executor/gardenhealth"
	"github.com/cloudfoundry/dropsonde/metric_sender/fake"
	"github.com/cloudfoundry/dropsonde/metrics"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"

	"code.cloudfoundry.org/clock/fakeclock"
	fakeexecutor "code.cloudfoundry.org/executor/fakes"
	"code.cloudfoundry.org/executor/gardenhealth/fakegardenhealth"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/lagertest"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
)

var _ = Describe("Runner", func() {
	var (
		runner                          *gardenhealth.Runner
		process                         ifrit.Process
		logger                          *lagertest.TestLogger
		checker                         *fakegardenhealth.FakeChecker
		executorClient                  *fakeexecutor.FakeClient
		sender                          *fake.FakeMetricSender
		fakeClock                       *fakeclock.FakeClock
		checkInterval, emissionInterval time.Duration
		timeoutDuration                 time.Duration
	)

	BeforeEach(func() {
		logger = lagertest.NewTestLogger("test")
		checker = &fakegardenhealth.FakeChecker{}
		executorClient = &fakeexecutor.FakeClient{}
		fakeClock = fakeclock.NewFakeClock(time.Now())
		checkInterval = 2 * time.Minute
		timeoutDuration = 1 * time.Minute
		emissionInterval = 30 * time.Second

		sender = fake.NewFakeMetricSender()
		metrics.Initialize(sender, nil)
	})

	JustBeforeEach(func() {
		runner = gardenhealth.NewRunner(checkInterval, emissionInterval, timeoutDuration, logger, checker, executorClient, fakeClock)
		process = ifrit.Background(runner)
	})

	AfterEach(func() {
		ginkgomon.Interrupt(process)
	})

	Describe("Run", func() {
		Context("When garden is immediately unhealthy", func() {
			Context("because the health check fails", func() {
				var checkErr = gardenhealth.UnrecoverableError("nope")
				BeforeEach(func() {
					checker.HealthcheckReturns(checkErr)
					executorClient.HealthyReturns(false)
				})

				It("fails without becoming ready", func() {
					Eventually(process.Wait()).Should(Receive(Equal(checkErr)))
					Consistently(process.Ready()).ShouldNot(BeClosed())
				})

				It("emits a metric for unhealthy cell", func() {
					Eventually(process.Wait()).Should(Receive(Equal(checkErr)))
					Eventually(func() float64 {
						return sender.GetValue("UnhealthyCell").Value
					}).Should(Equal(float64(1)))
				})
			})

			Context("because the health check timed out", func() {
				var blockHealthcheck chan struct{}

				BeforeEach(func() {
					blockHealthcheck = make(chan struct{})
					checker.HealthcheckStub = func(lager.Logger) error {
						<-blockHealthcheck
						return nil
					}
				})

				JustBeforeEach(func() {
					fakeClock.WaitForWatcherAndIncrement(timeoutDuration)
				})

				AfterEach(func() {
					// Send to the channel to eliminate the race
					blockHealthcheck <- struct{}{}
					close(blockHealthcheck)
					blockHealthcheck = nil
				})

				It("fails without becoming ready", func() {
					Eventually(process.Wait()).Should(Receive(Equal(gardenhealth.HealthcheckTimeoutError{})))
					Consistently(process.Ready()).ShouldNot(BeClosed())
				})

				It("emits a metric for unhealthy cell", func() {
					Eventually(process.Wait()).Should(Receive(Equal(gardenhealth.HealthcheckTimeoutError{})))
					Eventually(func() float64 {
						return sender.GetValue("UnhealthyCell").Value
					}).Should(Equal(float64(1)))
				})
			})
		})

		Context("When garden is healthy", func() {
			BeforeEach(func() {
				executorClient.HealthyReturns(true)
			})

			It("sets healthy to true only once", func() {
				Eventually(executorClient.SetHealthyCallCount).Should(Equal(1))
				_, healthy := executorClient.SetHealthyArgsForCall(0)
				Expect(healthy).Should(Equal(true))
				Expect(executorClient.SetHealthyCallCount()).To(Equal(1))
			})

			It("continues to check at the correct interval", func() {
				Eventually(checker.HealthcheckCallCount).Should(Equal(1))
				Eventually(process.Ready()).Should(BeClosed())

				fakeClock.WaitForWatcherAndIncrement(checkInterval)
				Eventually(checker.HealthcheckCallCount).Should(Equal(2))
				Eventually(logger).Should(gbytes.Say("check-complete"))

				fakeClock.WaitForWatcherAndIncrement(checkInterval)
				Eventually(checker.HealthcheckCallCount).Should(Equal(3))
				Eventually(logger).Should(gbytes.Say("check-complete"))

				fakeClock.WaitForWatcherAndIncrement(checkInterval)
				Eventually(checker.HealthcheckCallCount).Should(Equal(4))
				Eventually(logger).Should(gbytes.Say("check-complete"))
			})

			It("emits a metric for healthy cell", func() {
				Eventually(func() float64 {
					return sender.GetValue("UnhealthyCell").Value
				}).Should(Equal(float64(0)))
			})
		})

		Context("when garden is intermittently healthy", func() {
			var checkErr = errors.New("nope")

			BeforeEach(func() {
				executorClient.HealthyReturns(true)
			})

			It("Sets healthy to false after it fails, then to true after success and emits respective metrics", func() {
				Eventually(executorClient.SetHealthyCallCount).Should(Equal(1))
				_, healthy := executorClient.SetHealthyArgsForCall(0)
				Expect(healthy).Should(Equal(true))
				Expect(sender.GetValue("UnhealthyCell").Value).To(Equal(float64(0)))

				checker.HealthcheckReturns(checkErr)
				executorClient.HealthyReturns(false)
				fakeClock.WaitForWatcherAndIncrement(checkInterval)

				Eventually(executorClient.SetHealthyCallCount).Should(Equal(2))
				_, healthy = executorClient.SetHealthyArgsForCall(1)
				Expect(healthy).Should(Equal(false))
				Expect(sender.GetValue("UnhealthyCell").Value).To(Equal(float64(1)))

				checker.HealthcheckReturns(nil)
				executorClient.HealthyReturns(true)
				fakeClock.WaitForWatcherAndIncrement(checkInterval)

				Eventually(executorClient.SetHealthyCallCount).Should(Equal(3))
				_, healthy = executorClient.SetHealthyArgsForCall(2)
				Expect(healthy).Should(Equal(true))
				Expect(sender.GetValue("UnhealthyCell").Value).To(Equal(float64(0)))
			})
		})

		Context("When the healthcheck times out", func() {
			var blockHealthcheck chan struct{}

			BeforeEach(func() {
				blockHealthcheck = make(chan struct{})
				checker.HealthcheckStub = func(lager.Logger) error {
					logger.Info("blocking")
					<-blockHealthcheck
					logger.Info("unblocking")
					return nil
				}
			})

			AfterEach(func() {
				close(blockHealthcheck)
			})

			It("sets the executor to unhealthy and emits the unhealthy metric", func() {
				Eventually(blockHealthcheck).Should(BeSent(struct{}{}))
				Eventually(executorClient.SetHealthyCallCount).Should(Equal(1))

				fakeClock.WaitForWatcherAndIncrement(checkInterval)
				Eventually(checker.HealthcheckCallCount).Should(Equal(2))

				fakeClock.WaitForWatcherAndIncrement(timeoutDuration)

				Eventually(executorClient.SetHealthyCallCount).Should(Equal(2))
				_, healthy := executorClient.SetHealthyArgsForCall(1)
				Expect(healthy).Should(Equal(false))
				Eventually(func() float64 {
					return sender.GetValue("UnhealthyCell").Value
				}).Should(Equal(float64(1)))
			})
		})

		Context("When the runner is signaled", func() {
			Context("during the initial health check", func() {
				var blockHealthcheck chan struct{}

				BeforeEach(func() {
					blockHealthcheck = make(chan struct{})
					checker.HealthcheckStub = func(lager.Logger) error {
						<-blockHealthcheck
						return nil
					}
				})

				JustBeforeEach(func() {
					process.Signal(os.Interrupt)
				})

				It("exits with no error", func() {
					Eventually(process.Wait()).Should(Receive(BeNil()))
				})
			})

			Context("After the initial health check", func() {
				It("exits imediately with no error", func() {
					Eventually(executorClient.SetHealthyCallCount).Should(Equal(1))

					process.Signal(os.Interrupt)
					Eventually(process.Wait()).Should(Receive(BeNil()))
				})
			})
		})

		Context("UnhealthyCell metric emission", func() {
			It("emits the UnhealthyCell every emitInterval", func() {
				Eventually(executorClient.HealthyCallCount).Should(Equal(1))
				Eventually(func() float64 {
					return sender.GetValue("UnhealthyCell").Value
				}).Should(Equal(float64(1)))

				executorClient.HealthyReturns(true)
				fakeClock.WaitForWatcherAndIncrement(emissionInterval)

				Eventually(executorClient.HealthyCallCount).Should(Equal(2))
				Eventually(func() float64 {
					return sender.GetValue("UnhealthyCell").Value
				}).Should(Equal(float64(0)))

				executorClient.HealthyReturns(false)
				fakeClock.WaitForWatcherAndIncrement(emissionInterval)

				Eventually(executorClient.HealthyCallCount).Should(Equal(3))
				Eventually(func() float64 {
					return sender.GetValue("UnhealthyCell").Value
				}).Should(Equal(float64(1)))
			})
		})
	})
})
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package componentconfig

import (
	"fmt"
	"net"

	utilnet "k8s.io/kubernetes/pkg/util/net"
)

// used for validating command line opts
// TODO(mikedanese): remove these when we remove command line flags

type IPVar struct {
	Val *string
}

func (v IPVar) Set(s string) error {
	if net.ParseIP(s) == nil {
		return fmt.Errorf("%q is not a valid IP address", s)
	}
	if v.Val == nil {
		// it's okay to panic here since this is programmer error
		panic("the string pointer passed into IPVar should not be nil")
	}
	*v.Val = s
	return nil
}

func (v IPVar) String() string {
	if v.Val == nil {
		return ""
	}
	return *v.Val
}

func (v IPVar) Type() string {
	return "ip"
}

func (m *ProxyMode) Set(s string) error {
	nm := ProxyMode(s)
	m = &nm
	return nil
}

func (m *ProxyMode) String() string {
	if m != nil {
		return string(*m)
	}
	return ""
}

func (m *ProxyMode) Type() string {
	return "ProxyMode"
}

type PortRangeVar struct {
	Val *string
}

func (v PortRangeVar) Set(s string) error {
	if _, err := utilnet.ParsePortRange(s); err != nil {
		return fmt.Errorf("%q is not a valid port range: %v", s, err)
	}
	if v.Val == nil {
		// it's okay to panic here since this is programmer error
		panic("the string pointer passed into PortRangeVar should not be nil")
	}
	*v.Val = s
	return nil
}

func (v PortRangeVar) String() string {
	if v.Val == nil {
		return ""
	}
	return *v.Val
}

func (v PortRangeVar) Type() string {
	return "port-range"
}
<commit_msg>componentconfig: fix proxy mode set func<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package componentconfig

import (
	"fmt"
	"net"

	utilnet "k8s.io/kubernetes/pkg/util/net"
)

// used for validating command line opts
// TODO(mikedanese): remove these when we remove command line flags

type IPVar struct {
	Val *string
}

func (v IPVar) Set(s string) error {
	if net.ParseIP(s) == nil {
		return fmt.Errorf("%q is not a valid IP address", s)
	}
	if v.Val == nil {
		// it's okay to panic here since this is programmer error
		panic("the string pointer passed into IPVar should not be nil")
	}
	*v.Val = s
	return nil
}

func (v IPVar) String() string {
	if v.Val == nil {
		return ""
	}
	return *v.Val
}

func (v IPVar) Type() string {
	return "ip"
}

func (m *ProxyMode) Set(s string) error {
	*m = ProxyMode(s)
	return nil
}

func (m *ProxyMode) String() string {
	if m != nil {
		return string(*m)
	}
	return ""
}

func (m *ProxyMode) Type() string {
	return "ProxyMode"
}

type PortRangeVar struct {
	Val *string
}

func (v PortRangeVar) Set(s string) error {
	if _, err := utilnet.ParsePortRange(s); err != nil {
		return fmt.Errorf("%q is not a valid port range: %v", s, err)
	}
	if v.Val == nil {
		// it's okay to panic here since this is programmer error
		panic("the string pointer passed into PortRangeVar should not be nil")
	}
	*v.Val = s
	return nil
}

func (v PortRangeVar) String() string {
	if v.Val == nil {
		return ""
	}
	return *v.Val
}

func (v PortRangeVar) Type() string {
	return "port-range"
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package diskmanagers

import (
	"fmt"
	"strings"

	"github.com/golang/glog"
	"github.com/vmware/govmomi/object"
	"github.com/vmware/govmomi/vim25/types"
	"golang.org/x/net/context"
	"k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib"
)

// vmDiskManager implements VirtualDiskProvider interface for creating volume using Virtual Machine Reconfigure approach
type vmDiskManager struct {
	diskPath      string
	volumeOptions *vclib.VolumeOptions
	vmOptions     *vclib.VMOptions
}

// Create implements Disk's Create interface
// Contains implementation of VM based Provisioning to provision disk with SPBM Policy or VSANStorageProfileData
func (vmdisk vmDiskManager) Create(ctx context.Context, datastore *vclib.Datastore) (err error) {
	if vmdisk.volumeOptions.SCSIControllerType == "" {
		vmdisk.volumeOptions.SCSIControllerType = vclib.PVSCSIControllerType
	}
	pbmClient, err := vclib.NewPbmClient(ctx, datastore.Client())
	if err != nil {
		glog.Errorf("Error occurred while creating new pbmClient, err: %+v", err)
		return err
	}

	if vmdisk.volumeOptions.StoragePolicyID == "" && vmdisk.volumeOptions.StoragePolicyName != "" {
		vmdisk.volumeOptions.StoragePolicyID, err = pbmClient.ProfileIDByName(ctx, vmdisk.volumeOptions.StoragePolicyName)
		if err != nil {
			glog.Errorf("Error occurred while getting Profile Id from Profile Name: %s, err: %+v", vmdisk.volumeOptions.StoragePolicyName, err)
			return err
		}
	}
	if vmdisk.volumeOptions.StoragePolicyID != "" {
		compatible, faultMessage, err := datastore.IsCompatibleWithStoragePolicy(ctx, vmdisk.volumeOptions.StoragePolicyID)
		if err != nil {
			glog.Errorf("Error occurred while checking datastore compatibility with storage policy id: %s, err: %+v", vmdisk.volumeOptions.StoragePolicyID, err)
			return err
		}

		if !compatible {
			glog.Errorf("Datastore: %s is not compatible with Policy: %s", datastore.Name(), vmdisk.volumeOptions.StoragePolicyName)
			return fmt.Errorf("User specified datastore is not compatible with the storagePolicy: %q. Failed with faults: %+q", vmdisk.volumeOptions.StoragePolicyName, faultMessage)
		}
	}

	storageProfileSpec := &types.VirtualMachineDefinedProfileSpec{}
	// Is PBM storage policy ID is present, set the storage spec profile ID,
	// else, set raw the VSAN policy string.
	if vmdisk.volumeOptions.StoragePolicyID != "" {
		storageProfileSpec.ProfileId = vmdisk.volumeOptions.StoragePolicyID
	} else if vmdisk.volumeOptions.VSANStorageProfileData != "" {
		// Check Datastore type - VSANStorageProfileData is only applicable to vSAN Datastore
		dsType, err := datastore.GetType(ctx)
		if err != nil {
			return err
		}
		if dsType != vclib.VSANDatastoreType {
			glog.Errorf("The specified datastore: %q is not a VSAN datastore", datastore.Name())
			return fmt.Errorf("The specified datastore: %q is not a VSAN datastore."+
				" The policy parameters will work only with VSAN Datastore."+
				" So, please specify a valid VSAN datastore in Storage class definition.", datastore.Name())
		}
		storageProfileSpec.ProfileId = ""
		storageProfileSpec.ProfileData = &types.VirtualMachineProfileRawData{
			ExtensionKey: "com.vmware.vim.sps",
			ObjectData:   vmdisk.volumeOptions.VSANStorageProfileData,
		}
	} else {
		glog.Errorf("Both volumeOptions.StoragePolicyID and volumeOptions.VSANStorageProfileData are not set. One of them should be set")
		return fmt.Errorf("Both volumeOptions.StoragePolicyID and volumeOptions.VSANStorageProfileData are not set. One of them should be set")
	}
	var dummyVM *vclib.VirtualMachine
	// Check if VM already exist in the folder.
	// If VM is already present, use it, else create a new dummy VM.
	dummyVMFullName := vclib.DummyVMPrefixName + "-" + vmdisk.volumeOptions.Name
	dummyVM, err = datastore.Datacenter.GetVMByPath(ctx, vmdisk.vmOptions.VMFolder.InventoryPath+"/"+dummyVMFullName)
	if err != nil {
		// Create a dummy VM
		glog.V(1).Info("Creating Dummy VM: %q", dummyVMFullName)
		dummyVM, err = vmdisk.createDummyVM(ctx, datastore.Datacenter, dummyVMFullName)
		if err != nil {
			glog.Errorf("Failed to create Dummy VM. err: %v", err)
			return err
		}
	}

	// Reconfigure the VM to attach the disk with the VSAN policy configured
	virtualMachineConfigSpec := types.VirtualMachineConfigSpec{}
	disk, _, err := dummyVM.CreateDiskSpec(ctx, vmdisk.diskPath, datastore, vmdisk.volumeOptions)
	if err != nil {
		glog.Errorf("Failed to create Disk Spec. err: %v", err)
		return err
	}
	deviceConfigSpec := &types.VirtualDeviceConfigSpec{
		Device:        disk,
		Operation:     types.VirtualDeviceConfigSpecOperationAdd,
		FileOperation: types.VirtualDeviceConfigSpecFileOperationCreate,
	}

	deviceConfigSpec.Profile = append(deviceConfigSpec.Profile, storageProfileSpec)
	virtualMachineConfigSpec.DeviceChange = append(virtualMachineConfigSpec.DeviceChange, deviceConfigSpec)
	fileAlreadyExist := false
	task, err := dummyVM.Reconfigure(ctx, virtualMachineConfigSpec)
	err = task.Wait(ctx)
	if err != nil {
		fileAlreadyExist = isAlreadyExists(vmdisk.diskPath, err)
		if fileAlreadyExist {
			//Skip error and continue to detach the disk as the disk was already created on the datastore.
			glog.V(vclib.LogLevel).Info("File: %v already exists", vmdisk.diskPath)
		} else {
			glog.Errorf("Failed to attach the disk to VM: %q with err: %+v", dummyVMFullName, err)
			return err
		}
	}
	// Detach the disk from the dummy VM.
	err = dummyVM.DetachDisk(ctx, vmdisk.diskPath)
	if err != nil {
		if vclib.DiskNotFoundErrMsg == err.Error() && fileAlreadyExist {
			// Skip error if disk was already detached from the dummy VM but still present on the datastore.
			glog.V(vclib.LogLevel).Info("File: %v is already detached", vmdisk.diskPath)
		} else {
			glog.Errorf("Failed to detach the disk: %q from VM: %q with err: %+v", vmdisk.diskPath, dummyVMFullName, err)
			return err
		}
	}
	//  Delete the dummy VM
	err = dummyVM.DeleteVM(ctx)
	if err != nil {
		glog.Errorf("Failed to destroy the vm: %q with err: %+v", dummyVMFullName, err)
	}
	return nil
}

func (vmdisk vmDiskManager) Delete(ctx context.Context, datastore *vclib.Datastore) error {
	return fmt.Errorf("vmDiskManager.Delete is not supported")
}

// CreateDummyVM create a Dummy VM at specified location with given name.
func (vmdisk vmDiskManager) createDummyVM(ctx context.Context, datacenter *vclib.Datacenter, vmName string) (*vclib.VirtualMachine, error) {
	// Create a virtual machine config spec with 1 SCSI adapter.
	virtualMachineConfigSpec := types.VirtualMachineConfigSpec{
		Name: vmName,
		Files: &types.VirtualMachineFileInfo{
			VmPathName: "[" + vmdisk.volumeOptions.Datastore + "]",
		},
		NumCPUs:  1,
		MemoryMB: 4,
		DeviceChange: []types.BaseVirtualDeviceConfigSpec{
			&types.VirtualDeviceConfigSpec{
				Operation: types.VirtualDeviceConfigSpecOperationAdd,
				Device: &types.ParaVirtualSCSIController{
					VirtualSCSIController: types.VirtualSCSIController{
						SharedBus: types.VirtualSCSISharingNoSharing,
						VirtualController: types.VirtualController{
							BusNumber: 0,
							VirtualDevice: types.VirtualDevice{
								Key: 1000,
							},
						},
					},
				},
			},
		},
	}

	task, err := vmdisk.vmOptions.VMFolder.CreateVM(ctx, virtualMachineConfigSpec, vmdisk.vmOptions.VMResourcePool, nil)
	if err != nil {
		glog.Errorf("Failed to create VM. err: %+v", err)
		return nil, err
	}

	dummyVMTaskInfo, err := task.WaitForResult(ctx, nil)
	if err != nil {
		glog.Errorf("Error occurred while waiting for create VM task result. err: %+v", err)
		return nil, err
	}

	vmRef := dummyVMTaskInfo.Result.(object.Reference)
	dummyVM := object.NewVirtualMachine(datacenter.Client(), vmRef.Reference())
	return &vclib.VirtualMachine{VirtualMachine: dummyVM, Datacenter: datacenter}, nil
}

// CleanUpDummyVMs deletes stale dummyVM's
func CleanUpDummyVMs(ctx context.Context, folder *vclib.Folder, dc *vclib.Datacenter) error {
	vmList, err := folder.GetVirtualMachines(ctx)
	if err != nil {
		glog.V(4).Infof("Failed to get virtual machines in the kubernetes cluster: %s, err: %+v", folder.InventoryPath, err)
		return err
	}
	if vmList == nil || len(vmList) == 0 {
		glog.Errorf("No virtual machines found in the kubernetes cluster: %s", folder.InventoryPath)
		return fmt.Errorf("No virtual machines found in the kubernetes cluster: %s", folder.InventoryPath)
	}
	var dummyVMList []*vclib.VirtualMachine
	// Loop through VM's in the Kubernetes cluster to find dummy VM's
	for _, vm := range vmList {
		vmName, err := vm.ObjectName(ctx)
		if err != nil {
			glog.V(4).Infof("Unable to get name from VM with err: %+v", err)
			continue
		}
		if strings.HasPrefix(vmName, vclib.DummyVMPrefixName) {
			vmObj := vclib.VirtualMachine{VirtualMachine: object.NewVirtualMachine(dc.Client(), vm.Reference()), Datacenter: dc}
			dummyVMList = append(dummyVMList, &vmObj)
		}
	}
	for _, vm := range dummyVMList {
		err = vm.DeleteVM(ctx)
		if err != nil {
			glog.V(4).Infof("Unable to delete dummy VM with err: %+v", err)
			continue
		}
	}
	return nil
}

func isAlreadyExists(path string, err error) bool {
	errorMessage := fmt.Sprintf("Cannot complete the operation because the file or folder %s already exists", path)
	if errorMessage == err.Error() {
		return true
	}
	return false
}
<commit_msg>Using hash/fnv to generate the vmName<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package diskmanagers

import (
	"fmt"
	"hash/fnv"
	"strings"

	"github.com/golang/glog"
	"github.com/vmware/govmomi/object"
	"github.com/vmware/govmomi/vim25/types"
	"golang.org/x/net/context"
	"k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib"
)

// vmDiskManager implements VirtualDiskProvider interface for creating volume using Virtual Machine Reconfigure approach
type vmDiskManager struct {
	diskPath      string
	volumeOptions *vclib.VolumeOptions
	vmOptions     *vclib.VMOptions
}

// Create implements Disk's Create interface
// Contains implementation of VM based Provisioning to provision disk with SPBM Policy or VSANStorageProfileData
func (vmdisk vmDiskManager) Create(ctx context.Context, datastore *vclib.Datastore) (err error) {
	if vmdisk.volumeOptions.SCSIControllerType == "" {
		vmdisk.volumeOptions.SCSIControllerType = vclib.PVSCSIControllerType
	}
	pbmClient, err := vclib.NewPbmClient(ctx, datastore.Client())
	if err != nil {
		glog.Errorf("Error occurred while creating new pbmClient, err: %+v", err)
		return err
	}

	if vmdisk.volumeOptions.StoragePolicyID == "" && vmdisk.volumeOptions.StoragePolicyName != "" {
		vmdisk.volumeOptions.StoragePolicyID, err = pbmClient.ProfileIDByName(ctx, vmdisk.volumeOptions.StoragePolicyName)
		if err != nil {
			glog.Errorf("Error occurred while getting Profile Id from Profile Name: %s, err: %+v", vmdisk.volumeOptions.StoragePolicyName, err)
			return err
		}
	}
	if vmdisk.volumeOptions.StoragePolicyID != "" {
		compatible, faultMessage, err := datastore.IsCompatibleWithStoragePolicy(ctx, vmdisk.volumeOptions.StoragePolicyID)
		if err != nil {
			glog.Errorf("Error occurred while checking datastore compatibility with storage policy id: %s, err: %+v", vmdisk.volumeOptions.StoragePolicyID, err)
			return err
		}

		if !compatible {
			glog.Errorf("Datastore: %s is not compatible with Policy: %s", datastore.Name(), vmdisk.volumeOptions.StoragePolicyName)
			return fmt.Errorf("User specified datastore is not compatible with the storagePolicy: %q. Failed with faults: %+q", vmdisk.volumeOptions.StoragePolicyName, faultMessage)
		}
	}

	storageProfileSpec := &types.VirtualMachineDefinedProfileSpec{}
	// Is PBM storage policy ID is present, set the storage spec profile ID,
	// else, set raw the VSAN policy string.
	if vmdisk.volumeOptions.StoragePolicyID != "" {
		storageProfileSpec.ProfileId = vmdisk.volumeOptions.StoragePolicyID
	} else if vmdisk.volumeOptions.VSANStorageProfileData != "" {
		// Check Datastore type - VSANStorageProfileData is only applicable to vSAN Datastore
		dsType, err := datastore.GetType(ctx)
		if err != nil {
			return err
		}
		if dsType != vclib.VSANDatastoreType {
			glog.Errorf("The specified datastore: %q is not a VSAN datastore", datastore.Name())
			return fmt.Errorf("The specified datastore: %q is not a VSAN datastore."+
				" The policy parameters will work only with VSAN Datastore."+
				" So, please specify a valid VSAN datastore in Storage class definition.", datastore.Name())
		}
		storageProfileSpec.ProfileId = ""
		storageProfileSpec.ProfileData = &types.VirtualMachineProfileRawData{
			ExtensionKey: "com.vmware.vim.sps",
			ObjectData:   vmdisk.volumeOptions.VSANStorageProfileData,
		}
	} else {
		glog.Errorf("Both volumeOptions.StoragePolicyID and volumeOptions.VSANStorageProfileData are not set. One of them should be set")
		return fmt.Errorf("Both volumeOptions.StoragePolicyID and volumeOptions.VSANStorageProfileData are not set. One of them should be set")
	}
	var dummyVM *vclib.VirtualMachine
	// Check if VM already exist in the folder.
	// If VM is already present, use it, else create a new dummy VM.
	fnvHash := fnv.New32a()
	fnvHash.Write([]byte(vmdisk.volumeOptions.Name))
	dummyVMFullName := vclib.DummyVMPrefixName + "-" + fmt.Sprint(fnvHash.Sum32())
	dummyVM, err = datastore.Datacenter.GetVMByPath(ctx, vmdisk.vmOptions.VMFolder.InventoryPath+"/"+dummyVMFullName)
	if err != nil {
		// Create a dummy VM
		glog.V(1).Info("Creating Dummy VM: %q", dummyVMFullName)
		dummyVM, err = vmdisk.createDummyVM(ctx, datastore.Datacenter, dummyVMFullName)
		if err != nil {
			glog.Errorf("Failed to create Dummy VM. err: %v", err)
			return err
		}
	}

	// Reconfigure the VM to attach the disk with the VSAN policy configured
	virtualMachineConfigSpec := types.VirtualMachineConfigSpec{}
	disk, _, err := dummyVM.CreateDiskSpec(ctx, vmdisk.diskPath, datastore, vmdisk.volumeOptions)
	if err != nil {
		glog.Errorf("Failed to create Disk Spec. err: %v", err)
		return err
	}
	deviceConfigSpec := &types.VirtualDeviceConfigSpec{
		Device:        disk,
		Operation:     types.VirtualDeviceConfigSpecOperationAdd,
		FileOperation: types.VirtualDeviceConfigSpecFileOperationCreate,
	}

	deviceConfigSpec.Profile = append(deviceConfigSpec.Profile, storageProfileSpec)
	virtualMachineConfigSpec.DeviceChange = append(virtualMachineConfigSpec.DeviceChange, deviceConfigSpec)
	fileAlreadyExist := false
	task, err := dummyVM.Reconfigure(ctx, virtualMachineConfigSpec)
	err = task.Wait(ctx)
	if err != nil {
		fileAlreadyExist = isAlreadyExists(vmdisk.diskPath, err)
		if fileAlreadyExist {
			//Skip error and continue to detach the disk as the disk was already created on the datastore.
			glog.V(vclib.LogLevel).Info("File: %v already exists", vmdisk.diskPath)
		} else {
			glog.Errorf("Failed to attach the disk to VM: %q with err: %+v", dummyVMFullName, err)
			return err
		}
	}
	// Detach the disk from the dummy VM.
	err = dummyVM.DetachDisk(ctx, vmdisk.diskPath)
	if err != nil {
		if vclib.DiskNotFoundErrMsg == err.Error() && fileAlreadyExist {
			// Skip error if disk was already detached from the dummy VM but still present on the datastore.
			glog.V(vclib.LogLevel).Info("File: %v is already detached", vmdisk.diskPath)
		} else {
			glog.Errorf("Failed to detach the disk: %q from VM: %q with err: %+v", vmdisk.diskPath, dummyVMFullName, err)
			return err
		}
	}
	//  Delete the dummy VM
	err = dummyVM.DeleteVM(ctx)
	if err != nil {
		glog.Errorf("Failed to destroy the vm: %q with err: %+v", dummyVMFullName, err)
	}
	return nil
}

func (vmdisk vmDiskManager) Delete(ctx context.Context, datastore *vclib.Datastore) error {
	return fmt.Errorf("vmDiskManager.Delete is not supported")
}

// CreateDummyVM create a Dummy VM at specified location with given name.
func (vmdisk vmDiskManager) createDummyVM(ctx context.Context, datacenter *vclib.Datacenter, vmName string) (*vclib.VirtualMachine, error) {
	// Create a virtual machine config spec with 1 SCSI adapter.
	virtualMachineConfigSpec := types.VirtualMachineConfigSpec{
		Name: vmName,
		Files: &types.VirtualMachineFileInfo{
			VmPathName: "[" + vmdisk.volumeOptions.Datastore + "]",
		},
		NumCPUs:  1,
		MemoryMB: 4,
		DeviceChange: []types.BaseVirtualDeviceConfigSpec{
			&types.VirtualDeviceConfigSpec{
				Operation: types.VirtualDeviceConfigSpecOperationAdd,
				Device: &types.ParaVirtualSCSIController{
					VirtualSCSIController: types.VirtualSCSIController{
						SharedBus: types.VirtualSCSISharingNoSharing,
						VirtualController: types.VirtualController{
							BusNumber: 0,
							VirtualDevice: types.VirtualDevice{
								Key: 1000,
							},
						},
					},
				},
			},
		},
	}

	task, err := vmdisk.vmOptions.VMFolder.CreateVM(ctx, virtualMachineConfigSpec, vmdisk.vmOptions.VMResourcePool, nil)
	if err != nil {
		glog.Errorf("Failed to create VM. err: %+v", err)
		return nil, err
	}

	dummyVMTaskInfo, err := task.WaitForResult(ctx, nil)
	if err != nil {
		glog.Errorf("Error occurred while waiting for create VM task result. err: %+v", err)
		return nil, err
	}

	vmRef := dummyVMTaskInfo.Result.(object.Reference)
	dummyVM := object.NewVirtualMachine(datacenter.Client(), vmRef.Reference())
	return &vclib.VirtualMachine{VirtualMachine: dummyVM, Datacenter: datacenter}, nil
}

// CleanUpDummyVMs deletes stale dummyVM's
func CleanUpDummyVMs(ctx context.Context, folder *vclib.Folder, dc *vclib.Datacenter) error {
	vmList, err := folder.GetVirtualMachines(ctx)
	if err != nil {
		glog.V(4).Infof("Failed to get virtual machines in the kubernetes cluster: %s, err: %+v", folder.InventoryPath, err)
		return err
	}
	if vmList == nil || len(vmList) == 0 {
		glog.Errorf("No virtual machines found in the kubernetes cluster: %s", folder.InventoryPath)
		return fmt.Errorf("No virtual machines found in the kubernetes cluster: %s", folder.InventoryPath)
	}
	var dummyVMList []*vclib.VirtualMachine
	// Loop through VM's in the Kubernetes cluster to find dummy VM's
	for _, vm := range vmList {
		vmName, err := vm.ObjectName(ctx)
		if err != nil {
			glog.V(4).Infof("Unable to get name from VM with err: %+v", err)
			continue
		}
		if strings.HasPrefix(vmName, vclib.DummyVMPrefixName) {
			vmObj := vclib.VirtualMachine{VirtualMachine: object.NewVirtualMachine(dc.Client(), vm.Reference()), Datacenter: dc}
			dummyVMList = append(dummyVMList, &vmObj)
		}
	}
	for _, vm := range dummyVMList {
		err = vm.DeleteVM(ctx)
		if err != nil {
			glog.V(4).Infof("Unable to delete dummy VM with err: %+v", err)
			continue
		}
	}
	return nil
}

func isAlreadyExists(path string, err error) bool {
	errorMessage := fmt.Sprintf("Cannot complete the operation because the file or folder %s already exists", path)
	if errorMessage == err.Error() {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package runtime

import (
	"fmt"
	"os"
	"reflect"
	"sync"

	"github.com/apache/beam/sdks/go/pkg/beam/core/util/reflectx"
	"github.com/apache/beam/sdks/go/pkg/beam/core/util/symtab"
)

var (
	// Resolver is the accessible symbol resolver the runtime uses to find functions.
	Resolver SymbolResolver
	cache    = make(map[string]interface{})
	mu       sync.Mutex
)

func init() {
	// defer initialization of the default resolver. This way
	// the symbol table isn't read in unless strictly necessary.
	Resolver = &deferedResolver{initFn: initResolver}
}

type deferedResolver struct {
	initFn func() SymbolResolver
	r      SymbolResolver
	init   sync.Once
}

func (d *deferedResolver) Sym2Addr(name string) (uintptr, error) {
	d.init.Do(func() {
		d.r = d.initFn()
	})
	return d.r.Sym2Addr(name)
}

func initResolver() SymbolResolver {
	// First try the Linux location, since it's the most reliable.
	if r, err := symtab.New("/proc/self/exe"); err == nil {
		return r
	}
	// For other OS's this works in most cases we need.
	if r, err := symtab.New(os.Args[0]); err == nil {
		return r
	}
	return failResolver(false)
}

// SymbolResolver resolves a symbol to an unsafe address.
type SymbolResolver interface {
	// Sym2Addr returns the address pointer for a given symbol.
	Sym2Addr(string) (uintptr, error)
}

// RegisterFunction allows function registration. It is beneficial for performance
// and is needed for functions -- such as custom coders -- serialized during unit
// tests, where the underlying symbol table is not available. It should be called
// in init() only. Returns the external key for the function.
func RegisterFunction(fn interface{}) {
	if initialized {
		panic("Init hooks have already run. Register function during init() instead.")
	}

	key := reflectx.FunctionName(fn)
	if _, exists := cache[key]; exists {
		panic(fmt.Sprintf("Function %v already registred", key))
	}
	cache[key] = fn
}

// ResolveFunction resolves the runtime value of a given function by symbol name
// and type.
func ResolveFunction(name string, t reflect.Type) (interface{}, error) {
	mu.Lock()
	defer mu.Unlock()

	if val, exists := cache[name]; exists {
		return val, nil
	}

	ptr, err := Resolver.Sym2Addr(name)
	if err != nil {
		return 0, err
	}
	val := reflectx.LoadFunction(ptr, t)
	cache[name] = val
	return val, nil
}

type failResolver bool

func (p failResolver) Sym2Addr(name string) (uintptr, error) {
	return 0, fmt.Errorf("%v not found. Use runtime.RegisterFunction in unit tests", name)
}
<commit_msg>[BEAM-3612] Make function registration idempotent<commit_after>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package runtime

import (
	"fmt"
	"os"
	"reflect"
	"sync"

	"github.com/apache/beam/sdks/go/pkg/beam/core/util/reflectx"
	"github.com/apache/beam/sdks/go/pkg/beam/core/util/symtab"
)

var (
	// Resolver is the accessible symbol resolver the runtime uses to find functions.
	Resolver SymbolResolver
	cache    = make(map[string]interface{})
	mu       sync.Mutex
)

func init() {
	// defer initialization of the default resolver. This way
	// the symbol table isn't read in unless strictly necessary.
	Resolver = &deferedResolver{initFn: initResolver}
}

type deferedResolver struct {
	initFn func() SymbolResolver
	r      SymbolResolver
	init   sync.Once
}

func (d *deferedResolver) Sym2Addr(name string) (uintptr, error) {
	d.init.Do(func() {
		d.r = d.initFn()
	})
	return d.r.Sym2Addr(name)
}

func initResolver() SymbolResolver {
	// First try the Linux location, since it's the most reliable.
	if r, err := symtab.New("/proc/self/exe"); err == nil {
		return r
	}
	// For other OS's this works in most cases we need.
	if r, err := symtab.New(os.Args[0]); err == nil {
		return r
	}
	return failResolver(false)
}

// SymbolResolver resolves a symbol to an unsafe address.
type SymbolResolver interface {
	// Sym2Addr returns the address pointer for a given symbol.
	Sym2Addr(string) (uintptr, error)
}

// RegisterFunction allows function registration. It is beneficial for performance
// and is needed for functions -- such as custom coders -- serialized during unit
// tests, where the underlying symbol table is not available. It should be called
// in init() only. Returns the external key for the function.
func RegisterFunction(fn interface{}) {
	if initialized {
		panic("Init hooks have already run. Register function during init() instead.")
	}

	key := reflectx.FunctionName(fn)
	// If the function was registered already, the key and value will be the same anyway.
	cache[key] = fn
}

// ResolveFunction resolves the runtime value of a given function by symbol name
// and type.
func ResolveFunction(name string, t reflect.Type) (interface{}, error) {
	mu.Lock()
	defer mu.Unlock()

	if val, exists := cache[name]; exists {
		return val, nil
	}

	ptr, err := Resolver.Sym2Addr(name)
	if err != nil {
		return 0, err
	}
	val := reflectx.LoadFunction(ptr, t)
	cache[name] = val
	return val, nil
}

type failResolver bool

func (p failResolver) Sym2Addr(name string) (uintptr, error) {
	return 0, fmt.Errorf("%v not found. Use runtime.RegisterFunction in unit tests", name)
}
<|endoftext|>


<commit_before>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package cmd

import (
	"os"
	"path/filepath"

	"github.com/spf13/cobra"
	"github.com/webx-top/echo"
	"github.com/webx-top/echo/defaults"
	"github.com/webx-top/echo/middleware/language"

	"github.com/admpub/log"
	"github.com/admpub/nging/v4/application/cmd/bootconfig"
	"github.com/admpub/nging/v4/application/handler/setup"
	"github.com/admpub/nging/v4/application/library/config"
	"github.com/admpub/nging/v4/application/library/config/subconfig/sdb"
	"github.com/admpub/once"
)

// 静默安装
var InitDBConfig = &sdb.DB{
	Type:     `mysql`, // mysql / sqlite
	User:     `root`,
	Database: `nging`,
	Host:     `127.0.0.1:3306`,
}

var InitInstallConfig = &struct {
	Charset    string
	AdminUser  string
	AdminPass  string
	AdminEmail string
	Language   string // en / zh-cn
}{
	Charset:   sdb.MySQLDefaultCharset,
	AdminUser: `admin`,
	Language:  `zh-cn`,
}

var initCmd = &cobra.Command{
	Use:     "init",
	Short:   "Silent install",
	Example: filepath.Base(os.Args[0]) + " init [options]",
	RunE:    initRunE,
}

var translate *language.Translate
var translock once.Once

func initTranslate() {
	translate = BuildTranslator(config.DefaultConfig.Language)
}

func GetTranslator() *language.Translate {
	translock.Do(initTranslate)
	return translate
}

func ResetTranslator() {
	translock.Reset()
}

func BuildTranslator(c language.Config) *language.Translate {
	c.SetFSFunc(bootconfig.LangFSFunc)
	i18n := language.NewI18n(&c)
	tr := &language.Translate{}
	tr.Reset(InitInstallConfig.Language, i18n)
	return tr
}

func initRunE(cmd *cobra.Command, args []string) error {
	conf, err := config.InitConfig()
	if err != nil {
		return err
	}
	conf.AsDefault()
	ctx := defaults.NewMockContext()
	ctx.SetAuto(true)

	// 启用多语言支持
	ctx.SetTranslator(GetTranslator())

	ctx.Request().Header().Set(echo.HeaderAccept, echo.MIMETextPlain)
	ctx.Request().SetMethod(echo.POST)
	ctx.Request().Form().Set(`type`, InitDBConfig.Type)
	ctx.Request().Form().Set(`user`, InitDBConfig.User)
	ctx.Request().Form().Set(`host`, InitDBConfig.Host)
	ctx.Request().Form().Set(`password`, InitDBConfig.Password)
	ctx.Request().Form().Set(`database`, InitDBConfig.Database)
	ctx.Request().Form().Set(`prefix`, InitDBConfig.Prefix)
	ctx.Request().Form().Set(`charset`, InitInstallConfig.Charset)
	ctx.Request().Form().Set(`adminUser`, InitInstallConfig.AdminUser)
	ctx.Request().Form().Set(`adminPass`, InitInstallConfig.AdminPass)
	ctx.Request().Form().Set(`adminEmail`, InitInstallConfig.AdminEmail)
	//return ctx.Render(`index`, nil)
	err = setup.Setup(ctx)
	if err == nil {
		log.Okay(`Congratulations, this program has been installed successfully`)
	}
	return err
}

func init() {
	rootCmd.AddCommand(initCmd)

	initCmd.Flags().StringVar(&InitDBConfig.Type, "type", InitDBConfig.Type, "database type")
	initCmd.Flags().StringVar(&InitDBConfig.User, "user", InitDBConfig.User, "database user")
	initCmd.Flags().StringVar(&InitDBConfig.Host, "host", InitDBConfig.Host, "database host")
	initCmd.Flags().StringVar(&InitDBConfig.Password, "password", InitDBConfig.Password, "database password")
	initCmd.Flags().StringVar(&InitDBConfig.Database, "database", InitDBConfig.Database, "database name")
	initCmd.Flags().StringVar(&InitDBConfig.Prefix, "prefix", InitDBConfig.Prefix, "database table prefix")
	initCmd.Flags().StringVar(&InitInstallConfig.Charset, "charset", InitInstallConfig.Charset, "database table charset")
	initCmd.Flags().StringVar(&InitInstallConfig.AdminUser, "adminUser", InitInstallConfig.AdminUser, "administrator name")
	initCmd.Flags().StringVar(&InitInstallConfig.AdminPass, "adminPass", InitInstallConfig.AdminPass, "administrator password")
	initCmd.Flags().StringVar(&InitInstallConfig.AdminEmail, "adminEmail", InitInstallConfig.AdminEmail, "administrator e-mail")
}
<commit_msg>update<commit_after>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package cmd

import (
	"os"
	"path/filepath"

	"github.com/spf13/cobra"
	"github.com/webx-top/echo"
	"github.com/webx-top/echo/defaults"
	"github.com/webx-top/echo/middleware/language"

	"github.com/admpub/log"
	"github.com/admpub/nging/v4/application/cmd/bootconfig"
	"github.com/admpub/nging/v4/application/handler/setup"
	"github.com/admpub/nging/v4/application/library/config"
	"github.com/admpub/nging/v4/application/library/config/subconfig/sdb"
	"github.com/admpub/once"
)

// 静默安装
var InitDBConfig = &sdb.DB{
	Type:     `mysql`, // mysql / sqlite
	User:     `root`,
	Database: `nging`,
	Host:     `127.0.0.1:3306`,
}

var InitInstallConfig = &struct {
	Charset    string
	AdminUser  string
	AdminPass  string
	AdminEmail string
	Language   string // en / zh-cn
}{
	Charset:   sdb.MySQLDefaultCharset,
	AdminUser: `admin`,
	Language:  `zh-cn`,
}

var initCmd = &cobra.Command{
	Use:     "init",
	Short:   "Silent install",
	Example: filepath.Base(os.Args[0]) + " init [options]",
	RunE:    initRunE,
}

var translate *language.Translate
var translock once.Once

func initTranslate() {
	translate = BuildTranslator(config.DefaultConfig.Language)
}

func GetTranslator() *language.Translate {
	translock.Do(initTranslate)
	return translate
}

func ResetTranslator() {
	translock.Reset()
}

func BuildTranslator(c language.Config) *language.Translate {
	c.SetFSFunc(bootconfig.LangFSFunc)
	i18n := language.NewI18n(&c)
	tr := &language.Translate{}
	tr.Reset(InitInstallConfig.Language, i18n)
	return tr
}

func initRunE(cmd *cobra.Command, args []string) error {
	conf, err := config.InitConfig()
	if err != nil {
		return err
	}
	conf.AsDefault()
	ctx := defaults.NewMockContext()
	ctx.SetAuto(true)

	// 启用多语言支持
	ctx.SetTranslator(GetTranslator())

	ctx.Request().Header().Set(echo.HeaderAccept, echo.MIMETextPlain)
	ctx.Request().SetMethod(echo.POST)
	ctx.Request().Form().Set(`type`, InitDBConfig.Type)
	ctx.Request().Form().Set(`user`, InitDBConfig.User)
	ctx.Request().Form().Set(`host`, InitDBConfig.Host)
	ctx.Request().Form().Set(`password`, InitDBConfig.Password)
	ctx.Request().Form().Set(`database`, InitDBConfig.Database)
	ctx.Request().Form().Set(`prefix`, InitDBConfig.Prefix)
	ctx.Request().Form().Set(`charset`, InitInstallConfig.Charset)
	ctx.Request().Form().Set(`adminUser`, InitInstallConfig.AdminUser)
	ctx.Request().Form().Set(`adminPass`, InitInstallConfig.AdminPass)
	ctx.Request().Form().Set(`adminEmail`, InitInstallConfig.AdminEmail)
	//return ctx.Render(`index`, nil)
	err = setup.Setup(ctx)
	if err == nil {
		log.Okay(ctx.T(`Congratulations, this program has been installed successfully`))
	}
	return err
}

func init() {
	rootCmd.AddCommand(initCmd)

	initCmd.Flags().StringVar(&InitDBConfig.Type, "type", InitDBConfig.Type, "database type")
	initCmd.Flags().StringVar(&InitDBConfig.User, "user", InitDBConfig.User, "database user")
	initCmd.Flags().StringVar(&InitDBConfig.Host, "host", InitDBConfig.Host, "database host")
	initCmd.Flags().StringVar(&InitDBConfig.Password, "password", InitDBConfig.Password, "database password")
	initCmd.Flags().StringVar(&InitDBConfig.Database, "database", InitDBConfig.Database, "database name")
	initCmd.Flags().StringVar(&InitDBConfig.Prefix, "prefix", InitDBConfig.Prefix, "database table prefix")
	initCmd.Flags().StringVar(&InitInstallConfig.Charset, "charset", InitInstallConfig.Charset, "database table charset")
	initCmd.Flags().StringVar(&InitInstallConfig.AdminUser, "adminUser", InitInstallConfig.AdminUser, "administrator name")
	initCmd.Flags().StringVar(&InitInstallConfig.AdminPass, "adminPass", InitInstallConfig.AdminPass, "administrator password")
	initCmd.Flags().StringVar(&InitInstallConfig.AdminEmail, "adminEmail", InitInstallConfig.AdminEmail, "administrator e-mail")
}
<|endoftext|>


<commit_before>package base

import (
	"bytes"
	"errors"
	"fmt"
	"regexp"
	"time"
)

var ErrMustSetContains = errors.New("must set Contains or ContainsRe")
var ErrCallbackAlreadyTriggered = errors.New("callback set to 'OnlyOnce', but already triggered")
var ErrCallbackTimeout = errors.New("callback timeout")

type ReadCallbackOption func(callback *ReadCallback) error

func WithCallbackContains(contains string) ReadCallbackOption {
	return func(r *ReadCallback) error {
		r.Contains = contains

		return nil
	}
}

func WithCallbackContainsRe(contains string) ReadCallbackOption {
	return func(r *ReadCallback) error {
		r.ContainsRe = contains

		return nil
	}
}

func WithCallbackCaseInsensitive(i bool) ReadCallbackOption {
	return func(r *ReadCallback) error {
		r.CaseInsensitive = i

		return nil
	}
}

func WithCallbackMultiline(m bool) ReadCallbackOption {
	return func(r *ReadCallback) error {
		r.MultiLine = m

		return nil
	}
}

func WithCallbackResetOutput(reset bool) ReadCallbackOption {
	return func(r *ReadCallback) error {
		r.ResetOutput = reset

		return nil
	}
}

func WithCallbackOnlyOnce(o bool) ReadCallbackOption {
	return func(r *ReadCallback) error {
		r.OnlyOnce = o

		return nil
	}
}

func WithCallbackNextTimeout(t time.Duration) ReadCallbackOption {
	return func(r *ReadCallback) error {
		r.NextTimeout = t

		return nil
	}
}

func WithCallbackNextReadDelay(t time.Duration) ReadCallbackOption {
	return func(r *ReadCallback) error {
		r.NextReadDelay = t

		return nil
	}
}

func WithCallbackComplete(complete bool) ReadCallbackOption {
	return func(r *ReadCallback) error {
		r.Complete = complete

		return nil
	}
}

func WithCallbackName(name string) ReadCallbackOption {
	return func(r *ReadCallback) error {
		r.Name = name

		return nil
	}
}

func NewReadCallback(
	callback func(*Driver, string) error,
	options ...ReadCallbackOption,
) (*ReadCallback, error) {
	rc := &ReadCallback{
		Callback:           callback,
		Contains:           "",
		containsBytes:      nil,
		ContainsRe:         "",
		containsReCompiled: nil,
		CaseInsensitive:    true,
		MultiLine:          true,
		ResetOutput:        true,
		OnlyOnce:           false,
		NextTimeout:        0,
		NextReadDelay:      0,
		triggered:          false,
		Complete:           false,
		Name:               "",
	}

	for _, option := range options {
		err := option(rc)
		if err != nil {
			return nil, err
		}
	}

	if rc.Contains == "" && rc.ContainsRe == "" {
		return nil, ErrMustSetContains
	}

	return rc, nil
}

type ReadCallback struct {
	Callback           func(*Driver, string) error
	Contains           string
	containsBytes      []byte
	ContainsRe         string
	containsReCompiled *regexp.Regexp
	CaseInsensitive    bool
	MultiLine          bool
	// ResetOutput bool indicating if the output should be reset or not after callback execution.
	ResetOutput bool
	// OnlyOnce bool indicating if this callback should be executed only one time.
	OnlyOnce bool
	// NextTimout timeout value to use for the subsequent read loop - ignored if Complete is true.
	NextTimeout time.Duration
	// NextReadDelay is time to use for sleeps between reads for hte subsequent read loop.
	NextReadDelay time.Duration
	triggered     bool
	Complete      bool
	Name          string
}

func (r *ReadCallback) contains() []byte {
	if len(r.containsBytes) == 0 {
		r.containsBytes = []byte(r.Contains)

		if r.CaseInsensitive {
			r.containsBytes = bytes.ToLower(r.containsBytes)
		}
	}

	return r.containsBytes
}

func (r *ReadCallback) containsRe() *regexp.Regexp {
	if r.containsReCompiled == nil {
		flags := ""

		if r.CaseInsensitive && r.MultiLine {
			flags = "(?im)"
		} else if r.CaseInsensitive {
			flags = "(?i)"
		} else if r.MultiLine {
			flags = "(?m)"
		}

		r.containsReCompiled = regexp.MustCompile(fmt.Sprintf(`%s%s`, flags, r.ContainsRe))
	}

	return r.containsReCompiled
}

type readCallbackResult struct {
	i         int
	callbacks []*ReadCallback
	output    []byte
	err       error
}

func (d *Driver) executeCallback(
	i int,
	callbacks []*ReadCallback,
	output []byte,
	timeout,
	readDelay time.Duration) error {
	callback := callbacks[i]

	if callback.OnlyOnce {
		if callback.triggered {
			return ErrCallbackAlreadyTriggered
		}

		callback.triggered = true
	}

	err := callback.Callback(d, string(output))
	if err != nil {
		return err
	}

	if callback.Complete {
		return nil
	}

	if callback.ResetOutput {
		output = []byte{}
	}

	nextTimeout := timeout
	if callback.NextTimeout != 0 {
		nextTimeout = callback.NextTimeout
	}

	nextReadDelay := readDelay
	if callback.NextReadDelay != 0 {
		nextReadDelay = callback.NextReadDelay
	}

	return d.readWithCallbacks(callbacks, output, nextTimeout, nextReadDelay)
}

func (d *Driver) readWithCallbacks(
	callbacks []*ReadCallback,
	output []byte,
	timeout,
	readDelay time.Duration,
) error {
	c := make(chan *readCallbackResult)

	go func() {
		defer close(c)

		for {
			newOutput, err := d.Channel.Read()
			if err != nil {
				c <- &readCallbackResult{
					err: err,
				}

				return
			}

			output = append(output, newOutput...)

			for i, callback := range callbacks {
				o := output
				if callback.CaseInsensitive {
					o = bytes.ToLower(output)
				}

				if (callback.Contains != "" && bytes.Contains(o, callback.contains())) ||
					(callback.ContainsRe != "" && callback.containsRe().Match(o)) {
					c <- &readCallbackResult{
						i:         i,
						callbacks: callbacks,
						output:    output,
						err:       nil,
					}

					return
				}
			}

			time.Sleep(readDelay)
		}
	}()

	timer := time.NewTimer(timeout)

	select {
	case r := <-c:
		if r.err != nil {
			return r.err
		}

		return d.executeCallback(r.i, r.callbacks, r.output, timeout, readDelay)
	case <-timer.C:
		return ErrCallbackTimeout
	}
}

func (d *Driver) ReadWithCallbacks(
	callbacks []*ReadCallback,
	input string,
	timeout,
	readDelay time.Duration,
) error {
	if input != "" {
		err := d.Channel.WriteAndReturn([]byte(input), false)
		if err != nil {
			return err
		}
	}

	origTransportTimeout := d.Transport.BaseTransportArgs.TimeoutTransport
	d.Transport.BaseTransportArgs.TimeoutTransport = 0

	r := d.readWithCallbacks(callbacks, []byte{}, timeout, readDelay)

	d.Transport.BaseTransportArgs.TimeoutTransport = origTransportTimeout

	return r
}
<commit_msg>callback improvements<commit_after>package base

import (
	"bytes"
	"errors"
	"fmt"
	"regexp"
	"time"
)

var ErrMustSetContains = errors.New("must set Contains or ContainsRe")
var ErrCallbackAlreadyTriggered = errors.New("callback set to 'OnlyOnce', but already triggered")
var ErrCallbackTimeout = errors.New("callback timeout")

type ReadCallbackOption func(callback *ReadCallback) error

func WithCallbackContains(contains string) ReadCallbackOption {
	return func(r *ReadCallback) error {
		r.Contains = contains

		return nil
	}
}

func WithCallbackNotContains(notContains string) ReadCallbackOption {
	return func(r *ReadCallback) error {
		r.NotContains = notContains

		return nil
	}
}

func WithCallbackContainsRe(contains string) ReadCallbackOption {
	return func(r *ReadCallback) error {
		r.ContainsRe = contains

		return nil
	}
}

func WithCallbackCaseInsensitive(i bool) ReadCallbackOption {
	return func(r *ReadCallback) error {
		r.CaseInsensitive = i

		return nil
	}
}

func WithCallbackMultiline(m bool) ReadCallbackOption {
	return func(r *ReadCallback) error {
		r.MultiLine = m

		return nil
	}
}

func WithCallbackResetOutput(reset bool) ReadCallbackOption {
	return func(r *ReadCallback) error {
		r.ResetOutput = reset

		return nil
	}
}

func WithCallbackOnlyOnce(o bool) ReadCallbackOption {
	return func(r *ReadCallback) error {
		r.OnlyOnce = o

		return nil
	}
}

func WithCallbackNextTimeout(t time.Duration) ReadCallbackOption {
	return func(r *ReadCallback) error {
		r.NextTimeout = t

		return nil
	}
}

func WithCallbackNextReadDelay(t time.Duration) ReadCallbackOption {
	return func(r *ReadCallback) error {
		r.NextReadDelay = t

		return nil
	}
}

func WithCallbackComplete(complete bool) ReadCallbackOption {
	return func(r *ReadCallback) error {
		r.Complete = complete

		return nil
	}
}

func WithCallbackName(name string) ReadCallbackOption {
	return func(r *ReadCallback) error {
		r.Name = name

		return nil
	}
}

func NewReadCallback(
	callback func(*Driver, string) error,
	options ...ReadCallbackOption,
) (*ReadCallback, error) {
	rc := &ReadCallback{
		Callback:           callback,
		Contains:           "",
		containsBytes:      nil,
		ContainsRe:         "",
		containsReCompiled: nil,
		CaseInsensitive:    true,
		MultiLine:          true,
		ResetOutput:        true,
		OnlyOnce:           false,
		NextTimeout:        0,
		NextReadDelay:      0,
		triggered:          false,
		Complete:           false,
		Name:               "",
	}

	for _, option := range options {
		err := option(rc)
		if err != nil {
			return nil, err
		}
	}

	if rc.Contains == "" && rc.ContainsRe == "" {
		return nil, ErrMustSetContains
	}

	return rc, nil
}

type ReadCallback struct {
	Callback           func(*Driver, string) error
	Contains           string
	containsBytes      []byte
	NotContains        string
	notContainsBytes   []byte
	ContainsRe         string
	containsReCompiled *regexp.Regexp
	CaseInsensitive    bool
	MultiLine          bool
	// ResetOutput bool indicating if the output should be reset or not after callback execution.
	ResetOutput bool
	// OnlyOnce bool indicating if this callback should be executed only one time.
	OnlyOnce bool
	// NextTimout timeout value to use for the subsequent read loop - ignored if Complete is true.
	NextTimeout time.Duration
	// NextReadDelay is time to use for sleeps between reads for hte subsequent read loop.
	NextReadDelay time.Duration
	triggered     bool
	Complete      bool
	Name          string
}

func (r *ReadCallback) contains() []byte {
	if len(r.containsBytes) == 0 {
		r.containsBytes = []byte(r.Contains)

		if r.CaseInsensitive {
			r.containsBytes = bytes.ToLower(r.containsBytes)
		}
	}

	return r.containsBytes
}

func (r *ReadCallback) notContains() []byte {
	if len(r.notContainsBytes) == 0 {
		r.notContainsBytes = []byte(r.NotContains)

		if r.CaseInsensitive {
			r.notContainsBytes = bytes.ToLower(r.notContainsBytes)
		}
	}

	return r.notContainsBytes
}

func (r *ReadCallback) containsRe() *regexp.Regexp {
	if r.containsReCompiled == nil {
		flags := ""

		if r.CaseInsensitive && r.MultiLine {
			flags = "(?im)"
		} else if r.CaseInsensitive {
			flags = "(?i)"
		} else if r.MultiLine {
			flags = "(?m)"
		}

		r.containsReCompiled = regexp.MustCompile(fmt.Sprintf(`%s%s`, flags, r.ContainsRe))
	}

	return r.containsReCompiled
}

func (r *ReadCallback) check(o []byte) bool {
	if r.CaseInsensitive {
		o = bytes.ToLower(o)
	}

	if (r.Contains != "" && bytes.Contains(o, r.contains())) &&
		!(r.NotContains != "" && !bytes.Contains(o, r.notContains())) {
		return true
	}

	if (r.ContainsRe != "" && r.containsRe().Match(o)) &&
		!(r.NotContains != "" && !bytes.Contains(o, r.notContains())) {
		return true
	}

	return false
}

type readCallbackResult struct {
	i         int
	callbacks []*ReadCallback
	output    []byte
	err       error
}

func (d *Driver) executeCallback(
	i int,
	callbacks []*ReadCallback,
	output []byte,
	timeout,
	readDelay time.Duration) error {
	callback := callbacks[i]

	if callback.OnlyOnce {
		if callback.triggered {
			return ErrCallbackAlreadyTriggered
		}

		callback.triggered = true
	}

	err := callback.Callback(d, string(output))
	if err != nil {
		return err
	}

	if callback.Complete {
		return nil
	}

	if callback.ResetOutput {
		output = []byte{}
	}

	nextTimeout := timeout
	if callback.NextTimeout != 0 {
		nextTimeout = callback.NextTimeout
	}

	nextReadDelay := readDelay
	if callback.NextReadDelay != 0 {
		nextReadDelay = callback.NextReadDelay
	}

	return d.readWithCallbacks(callbacks, output, nextTimeout, nextReadDelay)
}

func (d *Driver) readWithCallbacks(
	callbacks []*ReadCallback,
	output []byte,
	timeout,
	readDelay time.Duration,
) error {
	c := make(chan *readCallbackResult)

	go func() {
		defer close(c)

		for {
			newOutput, err := d.Channel.Read()
			if err != nil {
				c <- &readCallbackResult{
					err: err,
				}

				return
			}

			output = append(output, newOutput...)

			for i, callback := range callbacks {
				if callback.check(output) {
					c <- &readCallbackResult{
						i:         i,
						callbacks: callbacks,
						output:    output,
						err:       nil,
					}

					return
				}
			}

			time.Sleep(readDelay)
		}
	}()

	timer := time.NewTimer(timeout)

	select {
	case r := <-c:
		if r.err != nil {
			return r.err
		}

		return d.executeCallback(r.i, r.callbacks, r.output, timeout, readDelay)
	case <-timer.C:
		return ErrCallbackTimeout
	}
}

func (d *Driver) ReadWithCallbacks(
	callbacks []*ReadCallback,
	input string,
	timeout,
	readDelay time.Duration,
) error {
	if input != "" {
		err := d.Channel.WriteAndReturn([]byte(input), false)
		if err != nil {
			return err
		}
	}

	origTransportTimeout := d.Transport.BaseTransportArgs.TimeoutTransport
	d.Transport.BaseTransportArgs.TimeoutTransport = 0

	r := d.readWithCallbacks(callbacks, []byte{}, timeout, readDelay)

	d.Transport.BaseTransportArgs.TimeoutTransport = origTransportTimeout

	return r
}
<|endoftext|>


<commit_before>package couch

import (
    "strings"
    "fmt"
    "os"
    "json"
    "bytes"
    "http"
    "net"
    "io/ioutil"
)

var (
    CouchDBHost = "localhost"
    CouchDBPort = "5984"
    CouchDBName = "exampledb"
)

//
// Helper and utility functions (private)
//

// Replaces all instances of from with to in s (quite inefficient right now)
func replace(s, from, to string) string {
    toks := strings.SplitAfter(s, from, 0)
    newstr := ""
    for i, tok := range toks {
        if i < len(toks)-1 {
            if !strings.HasSuffix(tok, from) {
                panic("problem in replace")
            }
            newtok := tok[0 : len(tok)-len(from)]
            newstr = newstr + newtok + to
        } else {
            newstr = newstr + tok
        }
    }
    return newstr
}

// Converts given URL to string containing the body of the response.
func url_to_string(url string) string {
    if r, _, err := http.Get(url); err == nil {
        b, err := ioutil.ReadAll(r.Body)
        r.Body.Close()
        if err == nil {
            return string(b)
        }
    }
    return ""
}

// Marshal given interface to JSON string
func to_JSON(p interface{}) (string, os.Error) {
    buf := new(bytes.Buffer)
    if err := json.Marshal(buf, p); err != nil {
        return "", err
    }
    return buf.String(), nil
}

// Unmarshal JSON string to given interface
func from_JSON(s string, p interface{}) os.Error {
    if ok, errtok := json.Unmarshal(s, p); !ok {
        return os.NewError(fmt.Sprintf("error unmarshaling: %s", errtok))
    }
    return nil
}

// Since the json pkg doesn't handle fields beginning with _, we need to
// convert "_id" and "_rev" to "Id" and "Rev" to extract that data.
func temp_hack_go_to_json(json_str string) string {
    json_str = replace(json_str, `"Id"`, `"_id"`)
    json_str = replace(json_str, `"Rev"`, `"_rev"`)
    return json_str
}

func temp_hack_json_to_go(json_str string) string {
    json_str = replace(json_str, `"_id"`, `"Id"`)
    json_str = replace(json_str, `"_rev"`, `"Rev"`)
    return json_str
}

type IdAndRev struct {
    Id  string
    Rev string
}

// Simply extract id and rev from a given JSON string (typically a document)
func extract_id_and_rev(json_str string) (string, string, os.Error) {
    // this assumes the temp replacement hack has already been applied
    id_rev := new(IdAndRev)
    if err := from_JSON(json_str, id_rev); err != nil {
        return "", "", err
    }
    return id_rev.Id, id_rev.Rev, nil
}

//
// Interface functions (public)
//

func CouchDBURL() string {
    return fmt.Sprintf("http://%s:%s/%s/", CouchDBHost, CouchDBPort, CouchDBName)
}

type InsertResponse struct {
    Ok  bool
    Id  string
    Rev string
}

// Inserts document to CouchDB, returning id and rev on success. The document
// interface may optionally specify an "Id" field.
func Insert(p interface{}) (string, string, os.Error) {
    body_type := "application/json"
    json_str, err := to_JSON(p)
    if err != nil {
        return "", "", err
    }
    json_str = temp_hack_go_to_json(json_str)

    r, err := http.Post(CouchDBURL(), body_type, bytes.NewBufferString(json_str))
    if err != nil {
        return "", "", err
    }

    b, err := ioutil.ReadAll(r.Body)
    r.Body.Close()
    if err != nil {
        return "", "", err
    }

    ir := new(InsertResponse)
    if err := from_JSON(string(b), ir); err != nil {
        return "", "", err
    }

    if !ir.Ok {
        return "", "", os.NewError(fmt.Sprintf("CouchDB returned not-OK: %v", ir))
    }

    return ir.Id, ir.Rev, nil
}

// Unmarshals the document matching id to the given interface, returning rev.
func Retrieve(id string, p interface{}) (string, os.Error) {
    if len(id) <= 0 {
        return "", os.NewError("no id specified")
    }

    json_str := url_to_string(fmt.Sprintf("%s%s", CouchDBURL(), id))
    json_str = temp_hack_json_to_go(json_str)
    _, rev, err := extract_id_and_rev(json_str)
    if err != nil {
        return "", err
    }

    return rev, from_JSON(json_str, p)
}

// Edits the given document, which must specify both id and rev fields (as "Id"
// and "Rev"), and returns the new rev.
func Edit(p interface{}) (string, os.Error) {
    _, rev, err := Insert(p)
    return rev, err
}

// Deletes document given by id and rev.
func Delete(id, rev string) os.Error {
    // Set up request
    var req http.Request
    req.Method = "DELETE"
    req.ProtoMajor = 1
    req.ProtoMinor = 1
    req.Close = true
    req.Header = map[string]string {
        "Content-Type": "application/json",
        "If-Match": rev,
    }
    req.TransferEncoding = []string{"chunked"}
    req.URL, _ = http.ParseURL(CouchDBURL() + id)

    // Make connection
    conn, err := net.Dial("tcp", "", CouchDBHost + ":" + CouchDBPort)
    if err != nil {
        return err
    }
    http_conn := http.NewClientConn(conn, nil)
    defer http_conn.Close()
    if err := http_conn.Write(&req); err != nil {
        return err
    }

    // Read response
    r, err := http_conn.Read()
    if r == nil {
        return os.NewError("no response")
    }
    if err != nil {
        return err
    }
    data, _ := ioutil.ReadAll(r.Body)
    r.Body.Close()
    ir := new(InsertResponse)
    if ok, _ := json.Unmarshal(string(data), ir); !ok {
        return os.NewError("error unmarshaling response")
    }
    if !ir.Ok {
        return os.NewError("CouchDB returned not-OK")
    }

    return nil
}

type Row struct {
    Id  string
    Key string
}

type KeyedViewResponse struct {
    Total_rows uint64
    Offset     uint64
    Rows       []Row
}

// Return array of document ids as returned by the given view, by given key.
func RetrieveIds(view, key string) []string {
    // view should be eg. "_design/my_foo/_view/my_bar"
    if len(view) <= 0 || len(key) <= 0 {
        return make([]string, 0)
    }

    full_url := fmt.Sprintf(`%s%s?key="%s"`, CouchDBURL(), view, key)
    json_str := url_to_string(full_url)
    kvr := new(KeyedViewResponse)
    if err := from_JSON(json_str, kvr); err != nil {
        return make([]string, 0)
    }

    ids := make([]string, len(kvr.Rows))
    for i, row := range kvr.Rows {
        ids[i] = row.Id
    }
    return ids
}
<commit_msg>Properly encode key="xyz" in RetrieveIds<commit_after>package couch

import (
    "strings"
    "fmt"
    "os"
    "json"
    "bytes"
    "http"
    "net"
    "io/ioutil"
)

var (
    CouchDBHost = "localhost"
    CouchDBPort = "5984"
    CouchDBName = "exampledb"
)

//
// Helper and utility functions (private)
//

// Replaces all instances of from with to in s (quite inefficient right now)
func replace(s, from, to string) string {
    toks := strings.SplitAfter(s, from, 0)
    newstr := ""
    for i, tok := range toks {
        if i < len(toks)-1 {
            if !strings.HasSuffix(tok, from) {
                panic("problem in replace")
            }
            newtok := tok[0 : len(tok)-len(from)]
            newstr = newstr + newtok + to
        } else {
            newstr = newstr + tok
        }
    }
    return newstr
}

// Converts given URL to string containing the body of the response.
func url_to_string(url string) string {
    if r, _, err := http.Get(url); err == nil {
        b, err := ioutil.ReadAll(r.Body)
        r.Body.Close()
        if err == nil {
            return string(b)
        }
    }
    return ""
}

// Marshal given interface to JSON string
func to_JSON(p interface{}) (string, os.Error) {
    buf := new(bytes.Buffer)
    if err := json.Marshal(buf, p); err != nil {
        return "", err
    }
    return buf.String(), nil
}

// Unmarshal JSON string to given interface
func from_JSON(s string, p interface{}) os.Error {
    if ok, errtok := json.Unmarshal(s, p); !ok {
        return os.NewError(fmt.Sprintf("error unmarshaling: %s", errtok))
    }
    return nil
}

// Since the json pkg doesn't handle fields beginning with _, we need to
// convert "_id" and "_rev" to "Id" and "Rev" to extract that data.
func temp_hack_go_to_json(json_str string) string {
    json_str = replace(json_str, `"Id"`, `"_id"`)
    json_str = replace(json_str, `"Rev"`, `"_rev"`)
    return json_str
}

func temp_hack_json_to_go(json_str string) string {
    json_str = replace(json_str, `"_id"`, `"Id"`)
    json_str = replace(json_str, `"_rev"`, `"Rev"`)
    return json_str
}

type IdAndRev struct {
    Id  string
    Rev string
}

// Simply extract id and rev from a given JSON string (typically a document)
func extract_id_and_rev(json_str string) (string, string, os.Error) {
    // this assumes the temp replacement hack has already been applied
    id_rev := new(IdAndRev)
    if err := from_JSON(json_str, id_rev); err != nil {
        return "", "", err
    }
    return id_rev.Id, id_rev.Rev, nil
}

//
// Interface functions (public)
//

func CouchDBURL() string {
    return fmt.Sprintf("http://%s:%s/%s/", CouchDBHost, CouchDBPort, CouchDBName)
}

type InsertResponse struct {
    Ok  bool
    Id  string
    Rev string
}

// Inserts document to CouchDB, returning id and rev on success. The document
// interface may optionally specify an "Id" field.
func Insert(p interface{}) (string, string, os.Error) {
    body_type := "application/json"
    json_str, err := to_JSON(p)
    if err != nil {
        return "", "", err
    }
    json_str = temp_hack_go_to_json(json_str)

    r, err := http.Post(CouchDBURL(), body_type, bytes.NewBufferString(json_str))
    if err != nil {
        return "", "", err
    }

    b, err := ioutil.ReadAll(r.Body)
    r.Body.Close()
    if err != nil {
        return "", "", err
    }

    ir := new(InsertResponse)
    if err := from_JSON(string(b), ir); err != nil {
        return "", "", err
    }

    if !ir.Ok {
        return "", "", os.NewError(fmt.Sprintf("CouchDB returned not-OK: %v", ir))
    }

    return ir.Id, ir.Rev, nil
}

// Unmarshals the document matching id to the given interface, returning rev.
func Retrieve(id string, p interface{}) (string, os.Error) {
    if len(id) <= 0 {
        return "", os.NewError("no id specified")
    }

    json_str := url_to_string(fmt.Sprintf("%s%s", CouchDBURL(), id))
    json_str = temp_hack_json_to_go(json_str)
    _, rev, err := extract_id_and_rev(json_str)
    if err != nil {
        return "", err
    }

    return rev, from_JSON(json_str, p)
}

// Edits the given document, which must specify both id and rev fields (as "Id"
// and "Rev"), and returns the new rev.
func Edit(p interface{}) (string, os.Error) {
    _, rev, err := Insert(p)
    return rev, err
}

// Deletes document given by id and rev.
func Delete(id, rev string) os.Error {
    // Set up request
    var req http.Request
    req.Method = "DELETE"
    req.ProtoMajor = 1
    req.ProtoMinor = 1
    req.Close = true
    req.Header = map[string]string {
        "Content-Type": "application/json",
        "If-Match": rev,
    }
    req.TransferEncoding = []string{"chunked"}
    req.URL, _ = http.ParseURL(CouchDBURL() + id)

    // Make connection
    conn, err := net.Dial("tcp", "", CouchDBHost + ":" + CouchDBPort)
    if err != nil {
        return err
    }
    http_conn := http.NewClientConn(conn, nil)
    defer http_conn.Close()
    if err := http_conn.Write(&req); err != nil {
        return err
    }

    // Read response
    r, err := http_conn.Read()
    if r == nil {
        return os.NewError("no response")
    }
    if err != nil {
        return err
    }
    data, _ := ioutil.ReadAll(r.Body)
    r.Body.Close()
    ir := new(InsertResponse)
    if ok, _ := json.Unmarshal(string(data), ir); !ok {
        return os.NewError("error unmarshaling response")
    }
    if !ir.Ok {
        return os.NewError("CouchDB returned not-OK")
    }

    return nil
}

type Row struct {
    Id  string
    Key string
}

type KeyedViewResponse struct {
    Total_rows uint64
    Offset     uint64
    Rows       []Row
}

// Return array of document ids as returned by the given view, by given key.
func RetrieveIds(view, key string) []string {
    // view should be eg. "_design/my_foo/_view/my_bar"
    if len(view) <= 0 || len(key) <= 0 {
        return make([]string, 0)
    }

    parameters = http.URLEncode(fmt.Sprintf(`key="%s"`, key))
    full_url := fmt.Sprintf("%s%s?%s", CouchDBURL(), view, parameters)
    json_str := url_to_string(full_url)
    kvr := new(KeyedViewResponse)
    if err := from_JSON(json_str, kvr); err != nil {
        return make([]string, 0)
    }

    ids := make([]string, len(kvr.Rows))
    for i, row := range kvr.Rows {
        ids[i] = row.Id
    }
    return ids
}
<|endoftext|>


<commit_before>package formats

import (
	"io"
	"sort"

	"github.com/alecthomas/template"
	"github.com/yarbelk/refasta/sequence"
)

// TNT formatter
type TNT struct {
	Title        string
	Sequences    map[string]map[string]sequence.Sequence
	MetaData     sequence.GMDSlice
	speciesNames []string
}

const tntNonInterleavedTemplateString = `xread
'{{ .Title }}'
{{ .Length }} {{ .NTaxa }}
{{ range $i, $taxon := .Taxa }}{{ $taxon.SpeciesName }} {{ $taxon.Sequence }}
{{ end }};`

var tntNonInterleavedTemplate = template.Must(template.New("TNT").Parse(tntNonInterleavedTemplateString))

type templateContext struct {
	Title         string
	Length, NTaxa int
	Taxa          []taxonData
}

type taxonData struct {
	SpeciesName string
	Sequence    sequence.SequenceData
}

const TNT_FORMAT = "tnt"

// Construct a species using a GMDSlice to order the gene sequences
func (t *TNT) PrintableTaxa() []taxonData {
	t.MetaData.Sort()
	var allSpecies []taxonData = make([]taxonData, 0, len(t.speciesNames))

	for _, n := range t.speciesNames {
		combinedSequences := make([]byte, 0, t.getTotalLength())
		for _, gmd := range t.MetaData {
			combinedSequences = append(combinedSequences, t.Sequences[gmd.Gene][n].Seq...)
		}
		allSpecies = append(allSpecies, taxonData{
			SpeciesName: sequence.Safe(n),
			Sequence:    combinedSequences,
		})
	}
	return allSpecies
}

// insertString into the place that would keep it uniquely and ordered ascending
func insertString(slice []string, s string) []string {
	i := sort.SearchStrings(slice, s)
	// Inserstion sort of the species names: builds up the list as a sorted list
	if i < len(slice) && slice[i] != s {
		// Species Name not in the list; insert it at i
		slice = append(slice[:i], append([]string{s}, slice[i:]...)...)
	} else if i == len(slice) {
		slice = append(slice, s)
	}
	return slice
}

// AddSequence (or multiple) to the internal sequence store.
func (t *TNT) AddSequence(seqs ...sequence.Sequence) {
	for _, seq := range seqs {
		if t.Sequences == nil {
			t.Sequences = make(map[string]map[string]sequence.Sequence)
		}
		if m, ok := t.Sequences[seq.Gene]; !ok || m == nil {
			t.Sequences[seq.Gene] = make(map[string]sequence.Sequence)
		}
		t.Sequences[seq.Gene][seq.Species] = seq
		t.speciesNames = insertString(t.speciesNames, seq.Species)
	}
}

// WriteSequences will collect up the sequences, verify their validity,
// and output a formated TNT file to the supplied writer
func (t *TNT) WriteSequences(writer io.Writer) error {
	gmd, err := t.GenerateMetaData()
	if err != nil {
		return err
	}
	gmd.Sort()
	t.MetaData = gmd
	allSpecies := t.PrintableTaxa()
	context := templateContext{
		Title:  t.Title,
		Length: t.getTotalLength(),
		NTaxa:  len(t.speciesNames),
		Taxa:   allSpecies,
	}
	return tntNonInterleavedTemplate.Execute(writer, context)
}

// GenerateMetaData will make sure that the sequences for the same
// gene sequence (or whatever sequence) are all the same length.
// Returns types of InvalidSequence with ErrNo
// MISSMATCHED_SEQUENCE_LENGTHS if they are no correct
// If they are correct, it will return a slice of the gene meta data
// GeneMetaData, sequence.GMDSlice
func (t *TNT) GenerateMetaData() (sequence.GMDSlice, error) {
	var expectedLen int
	geneMetaData := make(sequence.GMDSlice, 0, len(t.Sequences))

	for gene, _ := range t.Sequences {
		for i, name := range t.speciesNames {
			seq := t.Sequences[gene][name]
			if i == 0 {
				expectedLen = seq.Length
			} else if seq.Length != expectedLen {
				return nil, sequence.InvalidSequence{
					Message: "Sequences are not the Same length",
					Details: "None so far",
					Errno:   sequence.MISSMATCHED_SEQUENCE_LENGTHS,
				}
			}
		}
		geneMetaData = append(geneMetaData, sequence.GeneMetaData{
			Gene:          gene,
			Length:        expectedLen,
			NumberSpecies: len(t.Sequences[gene]),
		})
	}
	return geneMetaData, nil
}

// getTotalLength will return the combined length of all genes.  This should
// be the same for each species.
func (t *TNT) getTotalLength() (length int) {
	for _, gmd := range t.MetaData {
		length = length + gmd.Length
	}
	return
}
<commit_msg>Pull the xread tnt block into helper method<commit_after>package formats

import (
	"io"
	"sort"

	"github.com/alecthomas/template"
	"github.com/yarbelk/refasta/sequence"
)

// TNT formatter
type TNT struct {
	Title        string
	Sequences    map[string]map[string]sequence.Sequence
	MetaData     sequence.GMDSlice
	speciesNames []string
}

const tntNonInterleavedTemplateString = `xread
'{{ .Title }}'
{{ .Length }} {{ .NTaxa }}
{{ range $i, $taxon := .Taxa }}{{ $taxon.SpeciesName }} {{ $taxon.Sequence }}
{{ end }};`

var tntNonInterleavedTemplate = template.Must(template.New("TNT").Parse(tntNonInterleavedTemplateString))

type templateContext struct {
	Title         string
	Length, NTaxa int
	Taxa          []taxonData
}

type taxonData struct {
	SpeciesName string
	Sequence    sequence.SequenceData
}

const TNT_FORMAT = "tnt"

// Construct a species using a GMDSlice to order the gene sequences
func (t *TNT) PrintableTaxa() []taxonData {
	t.MetaData.Sort()
	var allSpecies []taxonData = make([]taxonData, 0, len(t.speciesNames))

	for _, n := range t.speciesNames {
		combinedSequences := make([]byte, 0, t.getTotalLength())
		for _, gmd := range t.MetaData {
			combinedSequences = append(combinedSequences, t.Sequences[gmd.Gene][n].Seq...)
		}
		allSpecies = append(allSpecies, taxonData{
			SpeciesName: sequence.Safe(n),
			Sequence:    combinedSequences,
		})
	}
	return allSpecies
}

// insertString into the place that would keep it uniquely and ordered ascending
func insertString(slice []string, s string) []string {
	i := sort.SearchStrings(slice, s)
	// Inserstion sort of the species names: builds up the list as a sorted list
	if i < len(slice) && slice[i] != s {
		// Species Name not in the list; insert it at i
		slice = append(slice[:i], append([]string{s}, slice[i:]...)...)
	} else if i == len(slice) {
		slice = append(slice, s)
	}
	return slice
}

// AddSequence (or multiple) to the internal sequence store.
func (t *TNT) AddSequence(seqs ...sequence.Sequence) {
	for _, seq := range seqs {
		if t.Sequences == nil {
			t.Sequences = make(map[string]map[string]sequence.Sequence)
		}
		if m, ok := t.Sequences[seq.Gene]; !ok || m == nil {
			t.Sequences[seq.Gene] = make(map[string]sequence.Sequence)
		}
		t.Sequences[seq.Gene][seq.Species] = seq
		t.speciesNames = insertString(t.speciesNames, seq.Species)
	}
}

/*
WriteXRead writes out the xread block; which contains the sequence
and taxa data

	xread
	taxa_1 CTAGC...
	taxa_2 TAGCA...
	;

*/
func (t *TNT) WriteXRead(writer io.Writer) error {
	allSpecies := t.PrintableTaxa()
	context := templateContext{
		Title:  t.Title,
		Length: t.getTotalLength(),
		NTaxa:  len(t.speciesNames),
		Taxa:   allSpecies,
	}
	return tntNonInterleavedTemplate.Execute(writer, context)
}

// WriteSequences will collect up the sequences, verify their validity,
// and output a formated TNT file to the supplied writer
func (t *TNT) WriteSequences(writer io.Writer) error {
	gmd, err := t.GenerateMetaData()

	if err != nil {
		return err
	}
	gmd.Sort()
	t.MetaData = gmd

	if err := t.WriteXRead(writer); err != nil {
		return err
	}

	return nil
}

// GenerateMetaData will make sure that the sequences for the same
// gene sequence (or whatever sequence) are all the same length.
// Returns types of InvalidSequence with ErrNo
// MISSMATCHED_SEQUENCE_LENGTHS if they are no correct
// If they are correct, it will return a slice of the gene meta data
// GeneMetaData, sequence.GMDSlice
func (t *TNT) GenerateMetaData() (sequence.GMDSlice, error) {
	var expectedLen int
	geneMetaData := make(sequence.GMDSlice, 0, len(t.Sequences))

	for gene, _ := range t.Sequences {
		for i, name := range t.speciesNames {
			seq := t.Sequences[gene][name]
			if i == 0 {
				expectedLen = seq.Length
			} else if seq.Length != expectedLen {
				return nil, sequence.InvalidSequence{
					Message: "Sequences are not the Same length",
					Details: "None so far",
					Errno:   sequence.MISSMATCHED_SEQUENCE_LENGTHS,
				}
			}
		}
		geneMetaData = append(geneMetaData, sequence.GeneMetaData{
			Gene:          gene,
			Length:        expectedLen,
			NumberSpecies: len(t.Sequences[gene]),
		})
	}
	return geneMetaData, nil
}

// getTotalLength will return the combined length of all genes.  This should
// be the same for each species.
func (t *TNT) getTotalLength() (length int) {
	for _, gmd := range t.MetaData {
		length = length + gmd.Length
	}
	return
}
<|endoftext|>


<commit_before>package models

import (
	"time"
)

// ProjectStatus is a type alias which will be used to create an enum of acceptable project status states.
type ProjectStatus string

// ProjectStatus pseudo-enum values
const (
	StatusPublished ProjectStatus = "published"

	Statuses = []ProjectStatus{StatusPublished}
)

// Errors pertaining to the data in a Project or operations on Projects.
var (
	ErrInvalidProjectStatus = fmt.Errorf("Project status must be one of the following: %s\n", strings.Join([]string(Statuses), ", "))
)

// Project contains information about a scanlation project, which has a human-readable name, a unique shorthand name,
// and a publishing status amongst other things.
type Project struct {
	Id          string        `json:"id"`
	Name        string        `json:"name"`
	Shorthand   string        `json:"projectName"`
	Description string        `json:"description"`
	Status      ProjectStatus `json:"status"`
	CreatedAt   time.Time     `json:"createdAt"`
}

// Validate checks that the "status" of the project is one of the accepted ProjectStatus values.
func (p Project) Validate() error {
	for _, status := range Statuses {
		if p.Status == status {
			return nil
		}
	}
	return ErrInvalidProjectStatus
}
<commit_msg>Rename some things to avoid name conflicts<commit_after>package models

import (
	"time"
)

// ProjectStatus is a type alias which will be used to create an enum of acceptable project status states.
type ProjectStatus string

// ProjectStatus pseudo-enum values
const (
	PStatusPublished ProjectStatus = "published"

	PStatuses = []ProjectStatus{StatusPublished}
)

// Errors pertaining to the data in a Project or operations on Projects.
var (
	ErrInvalidProjectStatus = fmt.Errorf("Project status must be one of the following: %s\n", strings.Join([]string(PStatuses), ", "))
)

// Project contains information about a scanlation project, which has a human-readable name, a unique shorthand name,
// and a publishing status amongst other things.
type Project struct {
	Id          string        `json:"id"`
	Name        string        `json:"name"`
	Shorthand   string        `json:"projectName"`
	Description string        `json:"description"`
	Status      ProjectStatus `json:"status"`
	CreatedAt   time.Time     `json:"createdAt"`
}

// Validate checks that the "status" of the project is one of the accepted ProjectStatus values.
func (p Project) Validate() error {
	for _, status := range PStatuses {
		if p.Status == status {
			return nil
		}
	}
	return ErrInvalidProjectStatus
}
<|endoftext|>