<commit_before>// Copyright 2014 Kelsey Hightower. All rights reserved.
// Use of this source code is governed by a BSD-style
// license found in the LICENSE file.

// Package memkv implements an in-memory key/value store.
package memkv

import (
	"errors"
	"path"
	"sort"
	"strings"
	"sync"
)

var ErrNotExist = errors.New("key does not exist")
var ErrNoMatch = errors.New("no keys match")

// A Store represents an in-memory key-value store safe for
// concurrent access.
type Store struct {
	FuncMap map[string]interface{}
	sync.RWMutex
	m map[string]KVPair
}

// New creates and initializes a new Store.
func New() Store {
	s := Store{m: make(map[string]KVPair)}
	s.FuncMap = map[string]interface{}{
		"exists": s.Exists,
		"ls":     s.List,
		"lsdir":  s.ListDir,
		"get":    s.Get,
		"gets":   s.GetAll,
		"getv":   s.GetValue,
		"getvs":  s.GetAllValues,
	}
	return s
}

// Delete deletes the KVPair associated with key.
func (s Store) Del(key string) {
	s.Lock()
	delete(s.m, key)
	s.Unlock()
}

// Exists checks for the existence of key in the store.
func (s Store) Exists(key string) bool {
	kv := s.Get(key)
	if kv.Value == "" {
		return false
	}
	return true
}

// Get gets the KVPair associated with key. If there is no KVPair
// associated with key, Get returns KVPair{}.
func (s Store) Get(key string) KVPair {
	s.RLock()
	kv := s.m[key]
	s.RUnlock()
	return kv
}

// GetAll returns a KVPair for all nodes with keys matching pattern.
// The syntax of patterns is the same as in path.Match.
func (s Store) GetAll(pattern string) KVPairs {
	ks := make(KVPairs, 0)
	s.RLock()
	defer s.RUnlock()
	for _, kv := range s.m {
		m, err := path.Match(pattern, kv.Key)
		if err != nil {
			return nil
		}
		if m {
			ks = append(ks, kv)
		}
	}
	if len(ks) == 0 {
		return nil
	}
	sort.Sort(ks)
	return ks
}

// GetAllKVs returns all KV-Pairs
func (s Store) GetAllKVs() KVPairs {
	ks := make(KVPairs, len(s.m))
	s.RLock()
	defer s.RUnlock()
	for _, kv := range s.m {
		ks = append(ks, kv)
	}
	return ks
}

// GetValue gets the value associated with key. If there are no values
// associated with key, GetValue returns "".
func (s Store) GetValue(key string, v ...string) string {
	defaultValue := ""
	if len(v) > 0 {
		defaultValue = v[0]
	}

	kv := s.Get(key)
	if kv.Key == "" {
		return defaultValue
	}
	return kv.Value
}

func (s Store) GetAllValues(pattern string) []string {
	vs := make([]string, 0)
	for _, kv := range s.GetAll(pattern) {
		vs = append(vs, kv.Value)
	}
	sort.Strings(vs)
	return vs
}

func (s Store) List(filePath string) []string {
	vs := make([]string, 0)
	m := make(map[string]bool)
	s.RLock()
	defer s.RUnlock()
	for _, kv := range s.m {
		if kv.Key == filePath {
			m[path.Base(kv.Key)] = true
			continue
		}
		if strings.HasPrefix(kv.Key, filePath) {
			m[strings.Split(stripKey(kv.Key, filePath), "/")[0]] = true
		}
	}
	for k := range m {
		vs = append(vs, k)
	}
	sort.Strings(vs)
	return vs
}

func (s Store) ListDir(filePath string) []string {
	vs := make([]string, 0)
	m := make(map[string]bool)
	s.RLock()
	defer s.RUnlock()
	for _, kv := range s.m {
		if strings.HasPrefix(kv.Key, filePath) {
			items := strings.Split(stripKey(kv.Key, filePath), "/")
			if len(items) < 2 {
				continue
			}
			m[items[0]] = true
		}
	}
	for k := range m {
		vs = append(vs, k)
	}
	sort.Strings(vs)
	return vs
}

// Set sets the KVPair entry associated with key to value.
func (s Store) Set(key string, value string) {
	s.Lock()
	s.m[key] = KVPair{key, value}
	s.Unlock()
}

func (s Store) Purge() {
	s.Lock()
	for k := range s.m {
		delete(s.m, k)
	}
	s.Unlock()
}

func stripKey(key, prefix string) string {
	return strings.TrimPrefix(strings.TrimPrefix(key, prefix), "/")
}
<commit_msg>add  getallkvs to the funcmap<commit_after>// Copyright 2014 Kelsey Hightower. All rights reserved.
// Use of this source code is governed by a BSD-style
// license found in the LICENSE file.

// Package memkv implements an in-memory key/value store.
package memkv

import (
	"errors"
	"path"
	"sort"
	"strings"
	"sync"
)

var ErrNotExist = errors.New("key does not exist")
var ErrNoMatch = errors.New("no keys match")

// A Store represents an in-memory key-value store safe for
// concurrent access.
type Store struct {
	FuncMap map[string]interface{}
	sync.RWMutex
	m map[string]KVPair
}

// New creates and initializes a new Store.
func New() Store {
	s := Store{m: make(map[string]KVPair)}
	s.FuncMap = map[string]interface{}{
		"exists":    s.Exists,
		"ls":        s.List,
		"lsdir":     s.ListDir,
		"get":       s.Get,
		"gets":      s.GetAll,
		"getallkvs": s.GetAllKVs,
		"getv":      s.GetValue,
		"getvs":     s.GetAllValues,
	}
	return s
}

// Delete deletes the KVPair associated with key.
func (s Store) Del(key string) {
	s.Lock()
	delete(s.m, key)
	s.Unlock()
}

// Exists checks for the existence of key in the store.
func (s Store) Exists(key string) bool {
	kv := s.Get(key)
	if kv.Value == "" {
		return false
	}
	return true
}

// Get gets the KVPair associated with key. If there is no KVPair
// associated with key, Get returns KVPair{}.
func (s Store) Get(key string) KVPair {
	s.RLock()
	kv := s.m[key]
	s.RUnlock()
	return kv
}

// GetAll returns a KVPair for all nodes with keys matching pattern.
// The syntax of patterns is the same as in path.Match.
func (s Store) GetAll(pattern string) KVPairs {
	ks := make(KVPairs, 0)
	s.RLock()
	defer s.RUnlock()
	for _, kv := range s.m {
		m, err := path.Match(pattern, kv.Key)
		if err != nil {
			return nil
		}
		if m {
			ks = append(ks, kv)
		}
	}
	if len(ks) == 0 {
		return nil
	}
	sort.Sort(ks)
	return ks
}

// GetAllKVs returns all KV-Pairs
func (s Store) GetAllKVs() KVPairs {
	ks := make(KVPairs, len(s.m))
	s.RLock()
	defer s.RUnlock()
	for _, kv := range s.m {
		ks = append(ks, kv)
	}
	return ks
}

// GetValue gets the value associated with key. If there are no values
// associated with key, GetValue returns "".
func (s Store) GetValue(key string, v ...string) string {
	defaultValue := ""
	if len(v) > 0 {
		defaultValue = v[0]
	}

	kv := s.Get(key)
	if kv.Key == "" {
		return defaultValue
	}
	return kv.Value
}

func (s Store) GetAllValues(pattern string) []string {
	vs := make([]string, 0)
	for _, kv := range s.GetAll(pattern) {
		vs = append(vs, kv.Value)
	}
	sort.Strings(vs)
	return vs
}

func (s Store) List(filePath string) []string {
	vs := make([]string, 0)
	m := make(map[string]bool)
	s.RLock()
	defer s.RUnlock()
	for _, kv := range s.m {
		if kv.Key == filePath {
			m[path.Base(kv.Key)] = true
			continue
		}
		if strings.HasPrefix(kv.Key, filePath) {
			m[strings.Split(stripKey(kv.Key, filePath), "/")[0]] = true
		}
	}
	for k := range m {
		vs = append(vs, k)
	}
	sort.Strings(vs)
	return vs
}

func (s Store) ListDir(filePath string) []string {
	vs := make([]string, 0)
	m := make(map[string]bool)
	s.RLock()
	defer s.RUnlock()
	for _, kv := range s.m {
		if strings.HasPrefix(kv.Key, filePath) {
			items := strings.Split(stripKey(kv.Key, filePath), "/")
			if len(items) < 2 {
				continue
			}
			m[items[0]] = true
		}
	}
	for k := range m {
		vs = append(vs, k)
	}
	sort.Strings(vs)
	return vs
}

// Set sets the KVPair entry associated with key to value.
func (s Store) Set(key string, value string) {
	s.Lock()
	s.m[key] = KVPair{key, value}
	s.Unlock()
}

func (s Store) Purge() {
	s.Lock()
	for k := range s.m {
		delete(s.m, k)
	}
	s.Unlock()
}

func stripKey(key, prefix string) string {
	return strings.TrimPrefix(strings.TrimPrefix(key, prefix), "/")
}
<|endoftext|>


<commit_before>// Copyright 2013 Jérôme Renard. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package utils

import (
	"os"
	"path/filepath"
)

func IsDir(d string) (bool, error) {
	fi, err := os.Stat(d)
	if err != nil {
		return false, err
	}

	return fi.IsDir(), nil
}

func HasFileName(f string) bool {
	ext := filepath.Ext(f)
	return len(ext) > 0
}
<commit_msg>Added utils.ToStringMap()<commit_after>// Copyright 2013 Jérôme Renard. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package utils

import (
	"errors"
	"os"
	"path/filepath"
	"reflect"
)

func IsDir(d string) (bool, error) {
	fi, err := os.Stat(d)
	if err != nil {
		return false, err
	}

	return fi.IsDir(), nil
}

func HasFileName(f string) bool {
	ext := filepath.Ext(f)
	return len(ext) > 0
}

func ToStringMap(m map[string]interface{}) (map[string]string, error) {
	value := reflect.ValueOf(m)
	if value.Kind() != reflect.Map {
		return map[string]string{}, errors.New("Not a map received")
	}

	keys := value.MapKeys()
	stringMap := make(map[string]string, len(keys))
	for _, key := range keys {
		stringMap[key.String()] = value.MapIndex(key).Elem().String()
	}

	return stringMap, nil
}
<|endoftext|>


<commit_before>package compiler_test

import (
	"errors"
	"os"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	fakebc "github.com/cloudfoundry/bosh-agent/agent/applier/bundlecollection/fakes"
	boshmodels "github.com/cloudfoundry/bosh-agent/agent/applier/models"
	fakepackages "github.com/cloudfoundry/bosh-agent/agent/applier/packages/fakes"
	fakecmdrunner "github.com/cloudfoundry/bosh-agent/agent/cmdrunner/fakes"
	. "github.com/cloudfoundry/bosh-agent/agent/compiler"
	fakeblobstore "github.com/cloudfoundry/bosh-agent/blobstore/fakes"
	fakecmd "github.com/cloudfoundry/bosh-agent/platform/commands/fakes"
	boshsys "github.com/cloudfoundry/bosh-agent/system"
	fakesys "github.com/cloudfoundry/bosh-agent/system/fakes"
)

type FakeCompileDirProvider struct {
	Dir string
}

func (cdp FakeCompileDirProvider) CompileDir() string { return cdp.Dir }

func getCompileArgs() (Package, []boshmodels.Package) {
	pkg := Package{
		BlobstoreID: "blobstore_id",
		Sha1:        "sha1",
		Name:        "pkg_name",
		Version:     "pkg_version",
	}

	pkgDeps := []boshmodels.Package{
		{
			Name:    "first_dep_name",
			Version: "first_dep_version",
			Source: boshmodels.Source{
				Sha1:        "first_dep_sha1",
				BlobstoreID: "first_dep_blobstore_id",
			},
		},
		{
			Name:    "sec_dep_name",
			Version: "sec_dep_version",
			Source: boshmodels.Source{
				Sha1:        "sec_dep_sha1",
				BlobstoreID: "sec_dep_blobstore_id",
			},
		},
	}

	return pkg, pkgDeps
}

func init() {
	Describe("concreteCompiler", func() {
		var (
			compiler       Compiler
			compressor     *fakecmd.FakeCompressor
			blobstore      *fakeblobstore.FakeBlobstore
			fs             *fakesys.FakeFileSystem
			runner         *fakecmdrunner.FakeFileLoggingCmdRunner
			packageApplier *fakepackages.FakeApplier
			packagesBc     *fakebc.FakeBundleCollection
		)

		BeforeEach(func() {
			compressor = fakecmd.NewFakeCompressor()
			blobstore = &fakeblobstore.FakeBlobstore{}
			fs = fakesys.NewFakeFileSystem()
			runner = fakecmdrunner.NewFakeFileLoggingCmdRunner()
			packageApplier = fakepackages.NewFakeApplier()
			packagesBc = fakebc.NewFakeBundleCollection()

			compiler = NewConcreteCompiler(
				compressor,
				blobstore,
				fs,
				runner,
				FakeCompileDirProvider{Dir: "/fake-compile-dir"},
				packageApplier,
				packagesBc,
			)
		})

		BeforeEach(func() {
			fs.MkdirAll("/fake-compile-dir", os.ModePerm)
		})

		Describe("Compile", func() {
			var (
				bundle  *fakebc.FakeBundle
				pkg     Package
				pkgDeps []boshmodels.Package
			)

			BeforeEach(func() {
				bundle = packagesBc.FakeGet(boshmodels.Package{
					Name:    "pkg_name",
					Version: "pkg_version",
				})

				bundle.InstallPath = "/fake-dir/data/packages/pkg_name/pkg_version"
				bundle.EnablePath = "/fake-dir/packages/pkg_name"

				compressor.CompressFilesInDirTarballPath = "/tmp/compressed-compiled-package"

				pkg, pkgDeps = getCompileArgs()
			})

			It("returns blob id and sha1 of created compiled package", func() {
				blobstore.CreateBlobID = "fake-blob-id"
				blobstore.CreateFingerprint = "fake-blob-sha1"

				blobID, sha1, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).ToNot(HaveOccurred())

				Expect(blobID).To(Equal("fake-blob-id"))
				Expect(sha1).To(Equal("fake-blob-sha1"))
			})

			It("cleans up all packages before applying dependent packages", func() {
				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).ToNot(HaveOccurred())
				Expect(packageApplier.ActionsCalled).To(Equal([]string{"KeepOnly", "Apply", "Apply"}))
				Expect(packageApplier.KeptOnlyPackages).To(BeEmpty())
			})

			It("returns an error if cleaning up packages fails", func() {
				packageApplier.KeepOnlyErr = errors.New("fake-keep-only-error")

				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("fake-keep-only-error"))
			})

			It("fetches source package from blobstore without checking SHA1 by default because of Director bug", func() {
				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).ToNot(HaveOccurred())

				Expect(blobstore.GetBlobIDs[0]).To(Equal("blobstore_id"))
				Expect(blobstore.GetFingerprints[0]).To(Equal(""))
			})

			It("fetches source package from blobstore and checks SHA1 by default in future", func() {
				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).ToNot(HaveOccurred())

				Expect(blobstore.GetBlobIDs[0]).To(Equal("blobstore_id"))

				// Wait for some time fixing default SHA1 check to stay backwards compatible
				fixDeadline := time.Date(2014, time.November, 13, 6, 0, 0, 0, time.UTC)

				if time.Now().After(fixDeadline) {
					Expect(blobstore.GetFingerprints[0]).To(Equal("sha1"))
				}
			})

			It("returns an error if removing compile target directory during uncompression fails", func() {
				fs.RegisterRemoveAllError("/fake-compile-dir/pkg_name", errors.New("fake-remove-error"))

				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("fake-remove-error"))
			})

			It("returns an error if creating compile target directory during uncompression fails", func() {
				fs.RegisterMkdirAllError("/fake-compile-dir/pkg_name", errors.New("fake-mkdir-error"))

				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("fake-mkdir-error"))
			})

			It("returns an error if removing temporary compile target directory during uncompression fails", func() {
				fs.RegisterRemoveAllError("/fake-compile-dir/pkg_name-bosh-agent-unpack", errors.New("fake-remove-error"))

				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("fake-remove-error"))
			})

			It("returns an error if creating temporary compile target directory during uncompression fails", func() {
				fs.RegisterMkdirAllError("/fake-compile-dir/pkg_name-bosh-agent-unpack", errors.New("fake-mkdir-error"))

				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("fake-mkdir-error"))
			})

			It("installs dependent packages", func() {
				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).ToNot(HaveOccurred())
				Expect(packageApplier.AppliedPackages).To(Equal(pkgDeps))
			})

			It("extracts source package to compile dir", func() {
				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).ToNot(HaveOccurred())

				Expect(fs.FileExists("/fake-compile-dir/pkg_name")).To(BeTrue())
				Expect(compressor.DecompressFileToDirDirs[0]).To(Equal("/fake-compile-dir/pkg_name-bosh-agent-unpack"))
				Expect(compressor.DecompressFileToDirTarballPaths[0]).To(Equal(blobstore.GetFileName))

				Expect(fs.RenameOldPaths[0]).To(Equal("/fake-compile-dir/pkg_name-bosh-agent-unpack"))
				Expect(fs.RenameNewPaths[0]).To(Equal("/fake-compile-dir/pkg_name"))
			})

			It("installs, enables and later cleans up bundle", func() {
				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).ToNot(HaveOccurred())
				Expect(bundle.ActionsCalled).To(Equal([]string{
					"InstallWithoutContents",
					"Enable",
					"Disable",
					"Uninstall",
				}))
			})

			Context("when packaging script exists", func() {
				BeforeEach(func() {
					compressor.DecompressFileToDirCallBack = func() {
						fs.WriteFileString("/fake-compile-dir/pkg_name/packaging", "hi")
					}
				})

				It("runs packaging script ", func() {
					_, _, err := compiler.Compile(pkg, pkgDeps)
					Expect(err).ToNot(HaveOccurred())

					expectedCmd := boshsys.Command{
						Name: "bash",
						Args: []string{"-x", "packaging"},
						Env: map[string]string{
							"BOSH_COMPILE_TARGET":  "/fake-compile-dir/pkg_name",
							"BOSH_INSTALL_TARGET":  "/fake-dir/packages/pkg_name",
							"BOSH_PACKAGE_NAME":    "pkg_name",
							"BOSH_PACKAGE_VERSION": "pkg_version",
						},
						WorkingDir: "/fake-compile-dir/pkg_name",
					}

					Expect(len(runner.RunCommands)).To(Equal(1))
					Expect(runner.RunCommands[0]).To(Equal(expectedCmd))
					Expect(runner.RunCommandJobName).To(Equal("compilation"))
					Expect(runner.RunCommandTaskName).To(Equal("packaging"))
				})

				It("propagates the error from packaging script", func() {
					runner.RunCommandErr = errors.New("fake-packaging-error")

					_, _, err := compiler.Compile(pkg, pkgDeps)
					Expect(err).To(HaveOccurred())
					Expect(err.Error()).To(ContainSubstring("fake-packaging-error"))
				})
			})

			It("does not run packaging script when script does not exist", func() {
				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).ToNot(HaveOccurred())
				Expect(runner.RunCommands).To(BeEmpty())
			})

			It("compresses compiled package", func() {
				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).ToNot(HaveOccurred())
				Expect(compressor.CompressFilesInDirDir).To(Equal("/fake-dir/data/packages/pkg_name/pkg_version"))
			})

			It("uploads compressed package to blobstore", func() {
				compressor.CompressFilesInDirTarballPath = "/tmp/compressed-compiled-package"

				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).ToNot(HaveOccurred())
				Expect(blobstore.CreateFileNames[0]).To(Equal("/tmp/compressed-compiled-package"))
			})

			It("returs error if uploading compressed package fails", func() {
				blobstore.CreateErr = errors.New("fake-create-err")

				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("fake-create-err"))
			})

			It("cleans up compressed package after uploading it to blobstore", func() {
				var beforeCleanUpTarballPath, afterCleanUpTarballPath string

				blobstore.CreateCallBack = func() {
					beforeCleanUpTarballPath = compressor.CleanUpTarballPath
				}

				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).ToNot(HaveOccurred())

				// Compressed package is not cleaned up before blobstore upload
				Expect(beforeCleanUpTarballPath).To(Equal(""))

				// Deleted after it was uploaded
				afterCleanUpTarballPath = compressor.CleanUpTarballPath
				Expect(afterCleanUpTarballPath).To(Equal("/tmp/compressed-compiled-package"))
			})
		})
	})
}
<commit_msg>Delay sha1 test failure for another 6 months<commit_after>package compiler_test

import (
	"errors"
	"os"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	fakebc "github.com/cloudfoundry/bosh-agent/agent/applier/bundlecollection/fakes"
	boshmodels "github.com/cloudfoundry/bosh-agent/agent/applier/models"
	fakepackages "github.com/cloudfoundry/bosh-agent/agent/applier/packages/fakes"
	fakecmdrunner "github.com/cloudfoundry/bosh-agent/agent/cmdrunner/fakes"
	. "github.com/cloudfoundry/bosh-agent/agent/compiler"
	fakeblobstore "github.com/cloudfoundry/bosh-agent/blobstore/fakes"
	fakecmd "github.com/cloudfoundry/bosh-agent/platform/commands/fakes"
	boshsys "github.com/cloudfoundry/bosh-agent/system"
	fakesys "github.com/cloudfoundry/bosh-agent/system/fakes"
)

type FakeCompileDirProvider struct {
	Dir string
}

func (cdp FakeCompileDirProvider) CompileDir() string { return cdp.Dir }

func getCompileArgs() (Package, []boshmodels.Package) {
	pkg := Package{
		BlobstoreID: "blobstore_id",
		Sha1:        "sha1",
		Name:        "pkg_name",
		Version:     "pkg_version",
	}

	pkgDeps := []boshmodels.Package{
		{
			Name:    "first_dep_name",
			Version: "first_dep_version",
			Source: boshmodels.Source{
				Sha1:        "first_dep_sha1",
				BlobstoreID: "first_dep_blobstore_id",
			},
		},
		{
			Name:    "sec_dep_name",
			Version: "sec_dep_version",
			Source: boshmodels.Source{
				Sha1:        "sec_dep_sha1",
				BlobstoreID: "sec_dep_blobstore_id",
			},
		},
	}

	return pkg, pkgDeps
}

func init() {
	Describe("concreteCompiler", func() {
		var (
			compiler       Compiler
			compressor     *fakecmd.FakeCompressor
			blobstore      *fakeblobstore.FakeBlobstore
			fs             *fakesys.FakeFileSystem
			runner         *fakecmdrunner.FakeFileLoggingCmdRunner
			packageApplier *fakepackages.FakeApplier
			packagesBc     *fakebc.FakeBundleCollection
		)

		BeforeEach(func() {
			compressor = fakecmd.NewFakeCompressor()
			blobstore = &fakeblobstore.FakeBlobstore{}
			fs = fakesys.NewFakeFileSystem()
			runner = fakecmdrunner.NewFakeFileLoggingCmdRunner()
			packageApplier = fakepackages.NewFakeApplier()
			packagesBc = fakebc.NewFakeBundleCollection()

			compiler = NewConcreteCompiler(
				compressor,
				blobstore,
				fs,
				runner,
				FakeCompileDirProvider{Dir: "/fake-compile-dir"},
				packageApplier,
				packagesBc,
			)
		})

		BeforeEach(func() {
			fs.MkdirAll("/fake-compile-dir", os.ModePerm)
		})

		Describe("Compile", func() {
			var (
				bundle  *fakebc.FakeBundle
				pkg     Package
				pkgDeps []boshmodels.Package
			)

			BeforeEach(func() {
				bundle = packagesBc.FakeGet(boshmodels.Package{
					Name:    "pkg_name",
					Version: "pkg_version",
				})

				bundle.InstallPath = "/fake-dir/data/packages/pkg_name/pkg_version"
				bundle.EnablePath = "/fake-dir/packages/pkg_name"

				compressor.CompressFilesInDirTarballPath = "/tmp/compressed-compiled-package"

				pkg, pkgDeps = getCompileArgs()
			})

			It("returns blob id and sha1 of created compiled package", func() {
				blobstore.CreateBlobID = "fake-blob-id"
				blobstore.CreateFingerprint = "fake-blob-sha1"

				blobID, sha1, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).ToNot(HaveOccurred())

				Expect(blobID).To(Equal("fake-blob-id"))
				Expect(sha1).To(Equal("fake-blob-sha1"))
			})

			It("cleans up all packages before applying dependent packages", func() {
				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).ToNot(HaveOccurred())
				Expect(packageApplier.ActionsCalled).To(Equal([]string{"KeepOnly", "Apply", "Apply"}))
				Expect(packageApplier.KeptOnlyPackages).To(BeEmpty())
			})

			It("returns an error if cleaning up packages fails", func() {
				packageApplier.KeepOnlyErr = errors.New("fake-keep-only-error")

				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("fake-keep-only-error"))
			})

			It("fetches source package from blobstore without checking SHA1 by default because of Director bug", func() {
				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).ToNot(HaveOccurred())

				Expect(blobstore.GetBlobIDs[0]).To(Equal("blobstore_id"))
				Expect(blobstore.GetFingerprints[0]).To(Equal(""))
			})

			It("fetches source package from blobstore and checks SHA1 by default in future", func() {
				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).ToNot(HaveOccurred())

				Expect(blobstore.GetBlobIDs[0]).To(Equal("blobstore_id"))

				// Do not implement SHA1 check in order to not break deployments for current users
				fixDeadline := time.Date(2015, time.May, 13, 6, 0, 0, 0, time.UTC)

				if time.Now().After(fixDeadline) {
					Expect(blobstore.GetFingerprints[0]).To(Equal("sha1"))
				}
			})

			It("returns an error if removing compile target directory during uncompression fails", func() {
				fs.RegisterRemoveAllError("/fake-compile-dir/pkg_name", errors.New("fake-remove-error"))

				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("fake-remove-error"))
			})

			It("returns an error if creating compile target directory during uncompression fails", func() {
				fs.RegisterMkdirAllError("/fake-compile-dir/pkg_name", errors.New("fake-mkdir-error"))

				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("fake-mkdir-error"))
			})

			It("returns an error if removing temporary compile target directory during uncompression fails", func() {
				fs.RegisterRemoveAllError("/fake-compile-dir/pkg_name-bosh-agent-unpack", errors.New("fake-remove-error"))

				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("fake-remove-error"))
			})

			It("returns an error if creating temporary compile target directory during uncompression fails", func() {
				fs.RegisterMkdirAllError("/fake-compile-dir/pkg_name-bosh-agent-unpack", errors.New("fake-mkdir-error"))

				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("fake-mkdir-error"))
			})

			It("installs dependent packages", func() {
				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).ToNot(HaveOccurred())
				Expect(packageApplier.AppliedPackages).To(Equal(pkgDeps))
			})

			It("extracts source package to compile dir", func() {
				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).ToNot(HaveOccurred())

				Expect(fs.FileExists("/fake-compile-dir/pkg_name")).To(BeTrue())
				Expect(compressor.DecompressFileToDirDirs[0]).To(Equal("/fake-compile-dir/pkg_name-bosh-agent-unpack"))
				Expect(compressor.DecompressFileToDirTarballPaths[0]).To(Equal(blobstore.GetFileName))

				Expect(fs.RenameOldPaths[0]).To(Equal("/fake-compile-dir/pkg_name-bosh-agent-unpack"))
				Expect(fs.RenameNewPaths[0]).To(Equal("/fake-compile-dir/pkg_name"))
			})

			It("installs, enables and later cleans up bundle", func() {
				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).ToNot(HaveOccurred())
				Expect(bundle.ActionsCalled).To(Equal([]string{
					"InstallWithoutContents",
					"Enable",
					"Disable",
					"Uninstall",
				}))
			})

			Context("when packaging script exists", func() {
				BeforeEach(func() {
					compressor.DecompressFileToDirCallBack = func() {
						fs.WriteFileString("/fake-compile-dir/pkg_name/packaging", "hi")
					}
				})

				It("runs packaging script ", func() {
					_, _, err := compiler.Compile(pkg, pkgDeps)
					Expect(err).ToNot(HaveOccurred())

					expectedCmd := boshsys.Command{
						Name: "bash",
						Args: []string{"-x", "packaging"},
						Env: map[string]string{
							"BOSH_COMPILE_TARGET":  "/fake-compile-dir/pkg_name",
							"BOSH_INSTALL_TARGET":  "/fake-dir/packages/pkg_name",
							"BOSH_PACKAGE_NAME":    "pkg_name",
							"BOSH_PACKAGE_VERSION": "pkg_version",
						},
						WorkingDir: "/fake-compile-dir/pkg_name",
					}

					Expect(len(runner.RunCommands)).To(Equal(1))
					Expect(runner.RunCommands[0]).To(Equal(expectedCmd))
					Expect(runner.RunCommandJobName).To(Equal("compilation"))
					Expect(runner.RunCommandTaskName).To(Equal("packaging"))
				})

				It("propagates the error from packaging script", func() {
					runner.RunCommandErr = errors.New("fake-packaging-error")

					_, _, err := compiler.Compile(pkg, pkgDeps)
					Expect(err).To(HaveOccurred())
					Expect(err.Error()).To(ContainSubstring("fake-packaging-error"))
				})
			})

			It("does not run packaging script when script does not exist", func() {
				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).ToNot(HaveOccurred())
				Expect(runner.RunCommands).To(BeEmpty())
			})

			It("compresses compiled package", func() {
				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).ToNot(HaveOccurred())
				Expect(compressor.CompressFilesInDirDir).To(Equal("/fake-dir/data/packages/pkg_name/pkg_version"))
			})

			It("uploads compressed package to blobstore", func() {
				compressor.CompressFilesInDirTarballPath = "/tmp/compressed-compiled-package"

				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).ToNot(HaveOccurred())
				Expect(blobstore.CreateFileNames[0]).To(Equal("/tmp/compressed-compiled-package"))
			})

			It("returs error if uploading compressed package fails", func() {
				blobstore.CreateErr = errors.New("fake-create-err")

				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("fake-create-err"))
			})

			It("cleans up compressed package after uploading it to blobstore", func() {
				var beforeCleanUpTarballPath, afterCleanUpTarballPath string

				blobstore.CreateCallBack = func() {
					beforeCleanUpTarballPath = compressor.CleanUpTarballPath
				}

				_, _, err := compiler.Compile(pkg, pkgDeps)
				Expect(err).ToNot(HaveOccurred())

				// Compressed package is not cleaned up before blobstore upload
				Expect(beforeCleanUpTarballPath).To(Equal(""))

				// Deleted after it was uploaded
				afterCleanUpTarballPath = compressor.CleanUpTarballPath
				Expect(afterCleanUpTarballPath).To(Equal("/tmp/compressed-compiled-package"))
			})
		})
	})
}
<|endoftext|>


<commit_before>// Copyright 2018 The Terraformer Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package terraform_utils

import (
	"bytes"
	"log"
	"sync"

	"github.com/GoogleCloudPlatform/terraformer/terraform_utils/provider_wrapper"

	"github.com/hashicorp/terraform/terraform"
)

type BaseResource struct {
	Tags map[string]string `json:"tags,omitempty"`
}

func NewTfState(resources []Resource) *terraform.State {
	tfstate := &terraform.State{
		Version:   terraform.StateVersion,
		TFVersion: terraform.VersionString(),
		Serial:    1,
	}
	outputs := map[string]*terraform.OutputState{}
	for _, r := range resources {
		for k, v := range r.Outputs {
			outputs[k] = v
		}
	}
	tfstate.Modules = []*terraform.ModuleState{
		{
			Path:      []string{"root"},
			Resources: map[string]*terraform.ResourceState{},
			Outputs:   outputs,
		},
	}
	for _, resource := range resources {
		resourceState := &terraform.ResourceState{
			Type:     resource.InstanceInfo.Type,
			Primary:  resource.InstanceState,
			Provider: "provider." + resource.Provider,
		}
		tfstate.Modules[0].Resources[resource.InstanceInfo.Type+"."+resource.ResourceName] = resourceState
	}
	return tfstate
}

func PrintTfState(resources []Resource) ([]byte, error) {
	state := NewTfState(resources)
	var buf bytes.Buffer
	err := terraform.WriteState(state, &buf)
	return buf.Bytes(), err
}

func RefreshResources(resources []Resource, providerName string, providerConfig map[string]interface{}) ([]Resource, error) {
	refreshedResources := []Resource{}
	input := make(chan *Resource, 100)
	provider, err := provider_wrapper.NewProviderWrapper(providerName, providerConfig)
	if err != nil {
		return refreshedResources, err
	}
	defer provider.Kill()
	var wg sync.WaitGroup
	for i := 0; i < 15; i++ {
		go RefreshResourceWorker(input, &wg, provider)
	}
	for i := range resources {
		wg.Add(1)
		input <- &resources[i]
	}
	wg.Wait()
	close(input)
	for _, r := range resources {
		if r.InstanceState != nil && r.InstanceState.ID != "" {
			refreshedResources = append(refreshedResources, r)
		}
	}
	return refreshedResources, nil
}

func RefreshResourceWorker(input chan *Resource, wg *sync.WaitGroup, provider *provider_wrapper.ProviderWrapper) {
	for r := range input {
		r.Refresh(provider)
		wg.Done()
	}
}

func IgnoreKeys(resourcesTypes []string, providerName string) map[string][]string {
	p, err := provider_wrapper.NewProviderWrapper(providerName, map[string]interface{}{})
	if err != nil {
		log.Println("plugin error:", err)
		return map[string][]string{}
	}
	defer p.Kill()
	readOnlyAttributes, err := p.GetReadOnlyAttributes(resourcesTypes)
	if err != nil {
		log.Println("plugin error:", err)
		return map[string][]string{}
	}
	return readOnlyAttributes
}
<commit_msg>Log refreshing status<commit_after>// Copyright 2018 The Terraformer Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package terraform_utils

import (
	"bytes"
	"log"
	"sync"

	"github.com/GoogleCloudPlatform/terraformer/terraform_utils/provider_wrapper"

	"github.com/hashicorp/terraform/terraform"
)

type BaseResource struct {
	Tags map[string]string `json:"tags,omitempty"`
}

func NewTfState(resources []Resource) *terraform.State {
	tfstate := &terraform.State{
		Version:   terraform.StateVersion,
		TFVersion: terraform.VersionString(),
		Serial:    1,
	}
	outputs := map[string]*terraform.OutputState{}
	for _, r := range resources {
		for k, v := range r.Outputs {
			outputs[k] = v
		}
	}
	tfstate.Modules = []*terraform.ModuleState{
		{
			Path:      []string{"root"},
			Resources: map[string]*terraform.ResourceState{},
			Outputs:   outputs,
		},
	}
	for _, resource := range resources {
		resourceState := &terraform.ResourceState{
			Type:     resource.InstanceInfo.Type,
			Primary:  resource.InstanceState,
			Provider: "provider." + resource.Provider,
		}
		tfstate.Modules[0].Resources[resource.InstanceInfo.Type+"."+resource.ResourceName] = resourceState
	}
	return tfstate
}

func PrintTfState(resources []Resource) ([]byte, error) {
	state := NewTfState(resources)
	var buf bytes.Buffer
	err := terraform.WriteState(state, &buf)
	return buf.Bytes(), err
}

func RefreshResources(resources []Resource, providerName string, providerConfig map[string]interface{}) ([]Resource, error) {
	refreshedResources := []Resource{}
	input := make(chan *Resource, 100)
	provider, err := provider_wrapper.NewProviderWrapper(providerName, providerConfig)
	if err != nil {
		return refreshedResources, err
	}
	defer provider.Kill()
	var wg sync.WaitGroup
	for i := 0; i < 15; i++ {
		go RefreshResourceWorker(input, &wg, provider)
	}
	for i := range resources {
		wg.Add(1)
		input <- &resources[i]
	}
	wg.Wait()
	close(input)
	for _, r := range resources {
		if r.InstanceState != nil && r.InstanceState.ID != "" {
			refreshedResources = append(refreshedResources, r)
		}
	}
	return refreshedResources, nil
}

func RefreshResourceWorker(input chan *Resource, wg *sync.WaitGroup, provider *provider_wrapper.ProviderWrapper) {
	for r := range input {
		log.Println("Refreshing state...", r.InstanceInfo.Id)
		r.Refresh(provider)
		wg.Done()
	}
}

func IgnoreKeys(resourcesTypes []string, providerName string) map[string][]string {
	p, err := provider_wrapper.NewProviderWrapper(providerName, map[string]interface{}{})
	if err != nil {
		log.Println("plugin error:", err)
		return map[string][]string{}
	}
	defer p.Kill()
	readOnlyAttributes, err := p.GetReadOnlyAttributes(resourcesTypes)
	if err != nil {
		log.Println("plugin error:", err)
		return map[string][]string{}
	}
	return readOnlyAttributes
}
<|endoftext|>


<commit_before>// Copyright ©2012 The bíogo Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package pals

import (
	"fmt"

	"github.com/biogo/biogo/feat"
	"github.com/biogo/store/interval"
)

var duplicatePair = fmt.Errorf("pals: attempt to add duplicate feature pair to pile")

// Note Location must be comparable according to http://golang.org/ref/spec#Comparison_operators.
type pileInterval struct {
	id         uintptr
	start, end int
	pile       *Pile
	location   feat.Feature
	images     []*Feature
	overlap    int
}

func (i *pileInterval) Overlap(b interval.IntRange) bool {
	return i.end-i.overlap >= b.Start && i.start <= b.End-i.overlap
}
func (i *pileInterval) ID() uintptr { return i.id }
func (i *pileInterval) Range() interval.IntRange {
	return interval.IntRange{Start: i.start + i.overlap, End: i.end - i.overlap}
}

// A Piler performs the aggregation of feature pairs according to the description in section 2.3
// of Edgar and Myers (2005) using an interval tree, giving O(nlogn) time but better space complexity
// and flexibility with feature overlap.
type Piler struct {
	intervals map[feat.Feature]*interval.IntTree
	seen      map[[2]sf]struct{}
	overlap   int
	piled     bool

	// next provides the next ID
	// for merged intervals. IDs
	// are unique across all intervals.
	next uintptr
}

type sf struct {
	loc  feat.Feature
	s, e int
}

// NewPiler creates a Piler object ready for piling feature pairs.
func NewPiler(overlap int) *Piler {
	return &Piler{
		intervals: make(map[feat.Feature]*interval.IntTree),
		seen:      make(map[[2]sf]struct{}),
		overlap:   overlap,
	}
}

// Add adds a feature pair to the piler incorporating the features into piles where appropriate.
func (p *Piler) Add(fp *Pair) error {
	a := sf{loc: fp.A.Location(), s: fp.A.Start(), e: fp.A.End()}
	b := sf{loc: fp.B.Location(), s: fp.B.Start(), e: fp.B.End()}
	ab, ba := [2]sf{a, b}, [2]sf{b, a}

	if _, ok := p.seen[ab]; ok {
		return duplicatePair
	}
	if _, ok := p.seen[ba]; ok {
		return duplicatePair
	}

	p.merge(&pileInterval{id: p.nextID(), start: fp.A.Start(), end: fp.A.End(), location: fp.A.Location(), images: []*Feature{fp.A}, overlap: p.overlap})
	p.merge(&pileInterval{id: p.nextID(), start: fp.B.Start(), end: fp.B.End(), location: fp.B.Location(), images: []*Feature{fp.B}, overlap: p.overlap})
	p.seen[ab] = struct{}{}

	return nil
}

func (p *Piler) nextID() uintptr {
	id := p.next
	p.next++
	return id
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func max(a, b int) int {
	if a > b {
		return a
	}
	return b
}

// merge merges an interval into the tree moving location meta data from the replaced intervals
// into the new interval.
func (p *Piler) merge(pi *pileInterval) {
	var (
		f  = true
		r  []interval.IntInterface
		qi = &pileInterval{start: pi.start, end: pi.end}
	)
	t, ok := p.intervals[pi.location]
	if !ok {
		t = &interval.IntTree{}
		p.intervals[pi.location] = t
	}
	t.DoMatching(
		func(e interval.IntInterface) (done bool) {
			r = append(r, e)
			iv := e.(*pileInterval)
			pi.images = append(pi.images, iv.images...)
			if f {
				pi.start = min(iv.start, pi.start)
				f = false
			}
			pi.end = max(iv.end, pi.end)
			return
		},
		qi,
	)
	for _, d := range r {
		t.Delete(d, false)
	}
	t.Insert(pi, false)
}

// A PairFilter is used to determine whether a Pair's images are included in a Pile.
type PairFilter func(*Pair) bool

// Piles returns a slice of piles determined by application of the filter function f to
// the feature pairs that have been added to the piler. Piles may be called more than once,
// but the piles returned in earlier invocations will be altered by subsequent calls.
func (p *Piler) Piles(f PairFilter) []*Pile {
	if !p.piled {
		for _, t := range p.intervals {
			t.Do(func(e interval.IntInterface) (done bool) {
				pa := e.(*pileInterval)
				if pa.pile == nil {
					pa.pile = &Pile{Loc: pa.location, From: pa.start, To: pa.end}
				}
				for _, im := range pa.images {
					if checkSanity {
						assertPileSanity(t, im, pa.pile)
					}
					im.Loc = pa.pile
				}
				return
			})
		}
		p.piled = true
	}

	var piles []*Pile
	for _, t := range p.intervals {
		t.Do(func(e interval.IntInterface) (done bool) {
			pa := e.(*pileInterval)
			pa.pile.Images = pa.pile.Images[:0]
			for _, im := range pa.images {
				if f != nil && !f(im.Pair) {
					continue
				}
				if checkSanity {
					assertPairSanity(im)
				}
				pa.pile.Images = append(pa.pile.Images, im)
			}
			piles = append(piles, pa.pile)
			return
		})
	}

	return piles
}

const checkSanity = false

func assertPileSanity(t *interval.IntTree, im *Feature, pi *Pile) {
	if im.Start() < pi.Start() || im.End() > pi.End() {
		panic(fmt.Sprintf("image extends beyond pile: %#v", im))
	}
	if foundPiles := t.Get(&pileInterval{start: im.Start(), end: im.End()}); len(foundPiles) > 1 {
		var containing int
		for _, pile := range foundPiles {
			r := pile.Range()
			if (r.Start <= im.Start() && r.End > im.End()) || (r.Start < im.Start() && r.End >= im.End()) {
				containing++
			}
		}
		if containing > 1 {
			panic(fmt.Sprintf("found too many piles for %#v", im))
		}
	}
}

func assertPairSanity(im *Feature) {
	if _, ok := im.Loc.(*Pile); !ok {
		panic(fmt.Sprintf("image not allocated to pile %#v", im))
	}
	if _, ok := im.Mate().Loc.(*Pile); !ok {
		panic(fmt.Sprintf("image mate not allocated to pile %#v", im.Mate()))
	}
}
<commit_msg>pals: add API for logging progress during piling<commit_after>// Copyright ©2012 The bíogo Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package pals

import (
	"fmt"
	"log"

	"github.com/biogo/biogo/feat"
	"github.com/biogo/store/interval"
)

var duplicatePair = fmt.Errorf("pals: attempt to add duplicate feature pair to pile")

// Note Location must be comparable according to http://golang.org/ref/spec#Comparison_operators.
type pileInterval struct {
	id         uintptr
	start, end int
	pile       *Pile
	location   feat.Feature
	images     []*Feature
	overlap    int
}

func (i *pileInterval) Overlap(b interval.IntRange) bool {
	return i.end-i.overlap >= b.Start && i.start <= b.End-i.overlap
}
func (i *pileInterval) ID() uintptr { return i.id }
func (i *pileInterval) Range() interval.IntRange {
	return interval.IntRange{Start: i.start + i.overlap, End: i.end - i.overlap}
}

// A Piler performs the aggregation of feature pairs according to the description in section 2.3
// of Edgar and Myers (2005) using an interval tree, giving O(nlogn) time but better space complexity
// and flexibility with feature overlap.
type Piler struct {
	// Logger logs pile construction during
	// Piles calls if non-nil.
	Logger *log.Logger
	// LogFreq specifies how frequently
	// log lines are witten if not zero.
	LogFreq int

	intervals map[feat.Feature]*interval.IntTree
	seen      map[[2]sf]struct{}
	overlap   int
	piled     bool

	// next provides the next ID
	// for merged intervals. IDs
	// are unique across all intervals.
	next uintptr
}

type sf struct {
	loc  feat.Feature
	s, e int
}

// NewPiler creates a Piler object ready for piling feature pairs.
func NewPiler(overlap int) *Piler {
	return &Piler{
		intervals: make(map[feat.Feature]*interval.IntTree),
		seen:      make(map[[2]sf]struct{}),
		overlap:   overlap,
	}
}

// Add adds a feature pair to the piler incorporating the features into piles where appropriate.
func (p *Piler) Add(fp *Pair) error {
	a := sf{loc: fp.A.Location(), s: fp.A.Start(), e: fp.A.End()}
	b := sf{loc: fp.B.Location(), s: fp.B.Start(), e: fp.B.End()}
	ab, ba := [2]sf{a, b}, [2]sf{b, a}

	if _, ok := p.seen[ab]; ok {
		return duplicatePair
	}
	if _, ok := p.seen[ba]; ok {
		return duplicatePair
	}

	p.merge(&pileInterval{id: p.nextID(), start: fp.A.Start(), end: fp.A.End(), location: fp.A.Location(), images: []*Feature{fp.A}, overlap: p.overlap})
	p.merge(&pileInterval{id: p.nextID(), start: fp.B.Start(), end: fp.B.End(), location: fp.B.Location(), images: []*Feature{fp.B}, overlap: p.overlap})
	p.seen[ab] = struct{}{}

	return nil
}

func (p *Piler) nextID() uintptr {
	id := p.next
	p.next++
	return id
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func max(a, b int) int {
	if a > b {
		return a
	}
	return b
}

// merge merges an interval into the tree moving location meta data from the replaced intervals
// into the new interval.
func (p *Piler) merge(pi *pileInterval) {
	var (
		f  = true
		r  []interval.IntInterface
		qi = &pileInterval{start: pi.start, end: pi.end}
	)
	t, ok := p.intervals[pi.location]
	if !ok {
		t = &interval.IntTree{}
		p.intervals[pi.location] = t
	}
	t.DoMatching(
		func(e interval.IntInterface) (done bool) {
			r = append(r, e)
			iv := e.(*pileInterval)
			pi.images = append(pi.images, iv.images...)
			if f {
				pi.start = min(iv.start, pi.start)
				f = false
			}
			pi.end = max(iv.end, pi.end)
			return
		},
		qi,
	)
	for _, d := range r {
		t.Delete(d, false)
	}
	t.Insert(pi, false)
}

// A PairFilter is used to determine whether a Pair's images are included in a Pile.
type PairFilter func(*Pair) bool

// Piles returns a slice of piles determined by application of the filter function f to
// the feature pairs that have been added to the piler. Piles may be called more than once,
// but the piles returned in earlier invocations will be altered by subsequent calls.
func (p *Piler) Piles(f PairFilter) []*Pile {
	var n int
	if !p.piled {
		for _, t := range p.intervals {
			t.Do(func(e interval.IntInterface) (done bool) {
				pa := e.(*pileInterval)
				if pa.pile == nil {
					pa.pile = &Pile{Loc: pa.location, From: pa.start, To: pa.end}
				}
				for _, im := range pa.images {
					if checkSanity {
						assertPileSanity(t, im, pa.pile)
					}
					im.Loc = pa.pile
				}
				return
			})
			n++
			if p.Logger != nil && p.LogFreq != 0 && n%p.LogFreq == 0 {
				p.Logger.Printf("piled %d intervals of %d", n, len(p.intervals))
			}
		}
		p.piled = true
	}

	n = 0
	var piles []*Pile
	for _, t := range p.intervals {
		t.Do(func(e interval.IntInterface) (done bool) {
			pa := e.(*pileInterval)
			pa.pile.Images = pa.pile.Images[:0]
			for _, im := range pa.images {
				if f != nil && !f(im.Pair) {
					continue
				}
				if checkSanity {
					assertPairSanity(im)
				}
				pa.pile.Images = append(pa.pile.Images, im)
			}
			piles = append(piles, pa.pile)
			return
		})
		n++
		if p.Logger != nil && p.LogFreq != 0 && n%p.LogFreq == 0 {
			p.Logger.Printf("filtered %d intervals of %d", n, len(p.intervals))
		}
	}

	return piles
}

const checkSanity = false

func assertPileSanity(t *interval.IntTree, im *Feature, pi *Pile) {
	if im.Start() < pi.Start() || im.End() > pi.End() {
		panic(fmt.Sprintf("image extends beyond pile: %#v", im))
	}
	if foundPiles := t.Get(&pileInterval{start: im.Start(), end: im.End()}); len(foundPiles) > 1 {
		var containing int
		for _, pile := range foundPiles {
			r := pile.Range()
			if (r.Start <= im.Start() && r.End > im.End()) || (r.Start < im.Start() && r.End >= im.End()) {
				containing++
			}
		}
		if containing > 1 {
			panic(fmt.Sprintf("found too many piles for %#v", im))
		}
	}
}

func assertPairSanity(im *Feature) {
	if _, ok := im.Loc.(*Pile); !ok {
		panic(fmt.Sprintf("image not allocated to pile %#v", im))
	}
	if _, ok := im.Mate().Loc.(*Pile); !ok {
		panic(fmt.Sprintf("image mate not allocated to pile %#v", im.Mate()))
	}
}
<|endoftext|>


<commit_before>package schema

import (
	"encoding/json"

	"github.com/zclconf/go-cty/cty"
	ctyjson "github.com/zclconf/go-cty/cty/json"

	"github.com/hashicorp/terraform/configs/configschema"
	"github.com/hashicorp/terraform/terraform"
)

// DiffFromValues takes the current state and desired state as cty.Values and
// derives a terraform.InstanceDiff to give to the legacy providers. This is
// used to take the states provided by the new ApplyResourceChange method and
// convert them to a state+diff required for the legacy Apply method.
func DiffFromValues(prior, planned cty.Value, res *Resource) (*terraform.InstanceDiff, error) {
	return diffFromValues(prior, planned, res, nil)
}

// diffFromValues takes an additional CustomizeDiffFunc, so we can generate our
// test fixtures from the legacy tests. In the new provider protocol the diff
// only needs to be created for the apply operation, and any customizations
// have already been done.
func diffFromValues(prior, planned cty.Value, res *Resource, cust CustomizeDiffFunc) (*terraform.InstanceDiff, error) {
	instanceState, err := res.ShimInstanceStateFromValue(prior)
	if err != nil {
		return nil, err
	}

	configSchema := res.CoreConfigSchema()

	cfg := terraform.NewResourceConfigShimmed(planned, configSchema)

	diff, err := schemaMap(res.Schema).Diff(instanceState, cfg, cust, nil, false)
	if err != nil {
		return nil, err
	}

	return diff, err
}

// ApplyDiff takes a cty.Value state and applies a terraform.InstanceDiff to
// get a new cty.Value state. This is used to convert the diff returned from
// the legacy provider Diff method to the state required for the new
// PlanResourceChange method.
func ApplyDiff(base cty.Value, d *terraform.InstanceDiff, schema *configschema.Block) (cty.Value, error) {
	return d.ApplyToValue(base, schema)
}

// StateValueToJSONMap converts a cty.Value to generic JSON map via the cty JSON
// encoding.
func StateValueToJSONMap(val cty.Value, ty cty.Type) (map[string]interface{}, error) {
	js, err := ctyjson.Marshal(val, ty)
	if err != nil {
		return nil, err
	}

	var m map[string]interface{}
	if err := json.Unmarshal(js, &m); err != nil {
		return nil, err
	}

	return m, nil
}

// JSONMapToStateValue takes a generic json map[string]interface{} and converts it
// to the specific type, ensuring that the values conform to the schema.
func JSONMapToStateValue(m map[string]interface{}, block *configschema.Block) (cty.Value, error) {
	var val cty.Value

	js, err := json.Marshal(m)
	if err != nil {
		return val, err
	}

	val, err = ctyjson.Unmarshal(js, block.ImpliedType())
	if err != nil {
		return val, err
	}

	return block.CoerceValue(val)
}

// StateValueFromInstanceState converts a terraform.InstanceState to a
// cty.Value as described by the provided cty.Type, and maintains the resource
// ID as the "id" attribute.
func StateValueFromInstanceState(is *terraform.InstanceState, ty cty.Type) (cty.Value, error) {
	return is.AttrsAsObjectValue(ty)
}
<commit_msg>LegacyResourceSchema to remove 0.12 features<commit_after>package schema

import (
	"encoding/json"

	"github.com/zclconf/go-cty/cty"
	ctyjson "github.com/zclconf/go-cty/cty/json"

	"github.com/hashicorp/terraform/configs/configschema"
	"github.com/hashicorp/terraform/terraform"
)

// DiffFromValues takes the current state and desired state as cty.Values and
// derives a terraform.InstanceDiff to give to the legacy providers. This is
// used to take the states provided by the new ApplyResourceChange method and
// convert them to a state+diff required for the legacy Apply method.
func DiffFromValues(prior, planned cty.Value, res *Resource) (*terraform.InstanceDiff, error) {
	return diffFromValues(prior, planned, res, nil)
}

// diffFromValues takes an additional CustomizeDiffFunc, so we can generate our
// test fixtures from the legacy tests. In the new provider protocol the diff
// only needs to be created for the apply operation, and any customizations
// have already been done.
func diffFromValues(prior, planned cty.Value, res *Resource, cust CustomizeDiffFunc) (*terraform.InstanceDiff, error) {
	instanceState, err := res.ShimInstanceStateFromValue(prior)
	if err != nil {
		return nil, err
	}

	configSchema := res.CoreConfigSchema()

	cfg := terraform.NewResourceConfigShimmed(planned, configSchema)

	diff, err := schemaMap(res.Schema).Diff(instanceState, cfg, cust, nil, false)
	if err != nil {
		return nil, err
	}

	return diff, err
}

// ApplyDiff takes a cty.Value state and applies a terraform.InstanceDiff to
// get a new cty.Value state. This is used to convert the diff returned from
// the legacy provider Diff method to the state required for the new
// PlanResourceChange method.
func ApplyDiff(base cty.Value, d *terraform.InstanceDiff, schema *configschema.Block) (cty.Value, error) {
	return d.ApplyToValue(base, schema)
}

// StateValueToJSONMap converts a cty.Value to generic JSON map via the cty JSON
// encoding.
func StateValueToJSONMap(val cty.Value, ty cty.Type) (map[string]interface{}, error) {
	js, err := ctyjson.Marshal(val, ty)
	if err != nil {
		return nil, err
	}

	var m map[string]interface{}
	if err := json.Unmarshal(js, &m); err != nil {
		return nil, err
	}

	return m, nil
}

// JSONMapToStateValue takes a generic json map[string]interface{} and converts it
// to the specific type, ensuring that the values conform to the schema.
func JSONMapToStateValue(m map[string]interface{}, block *configschema.Block) (cty.Value, error) {
	var val cty.Value

	js, err := json.Marshal(m)
	if err != nil {
		return val, err
	}

	val, err = ctyjson.Unmarshal(js, block.ImpliedType())
	if err != nil {
		return val, err
	}

	return block.CoerceValue(val)
}

// StateValueFromInstanceState converts a terraform.InstanceState to a
// cty.Value as described by the provided cty.Type, and maintains the resource
// ID as the "id" attribute.
func StateValueFromInstanceState(is *terraform.InstanceState, ty cty.Type) (cty.Value, error) {
	return is.AttrsAsObjectValue(ty)
}

// LegacyResourceSchema takes a *Resource and returns a deep copy with 0.12 specific
// features removed. This is used by the shims to get a configschema that
// directly matches the structure of the schema.Resource.
func LegacyResourceSchema(r *Resource) *Resource {
	if r == nil {
		return nil
	}
	// start with a shallow copy
	newResource := new(Resource)
	*newResource = *r
	newResource.Schema = map[string]*Schema{}

	for k, s := range r.Schema {
		newResource.Schema[k] = LegacySchema(s)
	}

	return newResource
}

// LegacySchema takes a *Schema and returns a deep copy with 0.12 specific
// features removed. This is used by the shims to get a configschema that
// directly matches the structure of the schema.Resource.
func LegacySchema(s *Schema) *Schema {
	if s == nil {
		return nil
	}
	// start with a shallow copy
	newSchema := new(Schema)
	*newSchema = *s
	newSchema.ConfigMode = SchemaConfigModeAuto
	newSchema.PromoteSingle = false
	newSchema.SkipCoreTypeCheck = false

	switch e := newSchema.Elem.(type) {
	case *Schema:
		newSchema.Elem = LegacySchema(e)
	case *Resource:
		newSchema.Elem = LegacyResourceSchema(e)
	}

	return newSchema
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"time"

	"github.com/codegangsta/cli"
	"github.com/xenolf/lego/acme"
)

func checkFolder(path string) error {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		return os.MkdirAll(path, 0700)
	}
	return nil
}

func setup(c *cli.Context) (*Configuration, *Account, *acme.Client) {
	err := checkFolder(c.GlobalString("path"))
	if err != nil {
		logger().Fatalf("Cound not check/create path: %s", err.Error())
	}

	conf := NewConfiguration(c)
	if len(c.GlobalString("email")) == 0 {
		logger().Fatal("You have to pass an account (email address) to the program using --email or -m")
	}

	//TODO: move to account struct? Currently MUST pass email.
	acc := NewAccount(c.GlobalString("email"), conf)

	client, err := acme.NewClient(c.GlobalString("server"), acc, conf.RsaBits())
	if err != nil {
		logger().Fatalf("Could not create client: %s", err.Error())
	}

	if len(c.GlobalStringSlice("exclude")) > 0 {
		client.ExcludeChallenges(conf.ExcludedSolvers())
	}

	if c.GlobalIsSet("http") {
		client.SetHTTPAddress(c.GlobalString("http"))
	}

	if c.GlobalIsSet("tls") {
		client.SetTLSAddress(c.GlobalString("tls"))
	}

	if c.GlobalIsSet("dns") {
		var err error
		var provider acme.ChallengeProvider
		switch c.GlobalString("dns") {
		case "cloudflare":
			provider, err = acme.NewDNSProviderCloudFlare("", "")
		case "digitalocean":
			authToken := os.Getenv("DO_AUTH_TOKEN")

			provider, err = acme.NewDNSProviderDigitalOcean(authToken)
		case "dnsimple":
			provider, err = acme.NewDNSProviderDNSimple("", "")
		case "route53":
			awsRegion := os.Getenv("AWS_REGION")
			provider, err = acme.NewDNSProviderRoute53("", "", awsRegion)
		case "rfc2136":
			nameserver := os.Getenv("RFC2136_NAMESERVER")
			zone := os.Getenv("RFC2136_ZONE")
			tsigAlgorithm := os.Getenv("RFC2136_TSIG_ALGORITHM")
			tsigKey := os.Getenv("RFC2136_TSIG_KEY")
			tsigSecret := os.Getenv("RFC2136_TSIG_SECRET")

			provider, err = acme.NewDNSProviderRFC2136(nameserver, zone, tsigAlgorithm, tsigKey, tsigSecret)
		case "manual":
			provider, err = acme.NewDNSProviderManual()
		}

		if err != nil {
			logger().Fatal(err)
		}

		client.SetChallengeProvider(acme.DNS01, provider)
	}

	return conf, acc, client
}

func saveCertRes(certRes acme.CertificateResource, conf *Configuration) {
	// We store the certificate, private key and metadata in different files
	// as web servers would not be able to work with a combined file.
	certOut := path.Join(conf.CertPath(), certRes.Domain+".crt")
	privOut := path.Join(conf.CertPath(), certRes.Domain+".key")
	metaOut := path.Join(conf.CertPath(), certRes.Domain+".json")

	err := ioutil.WriteFile(certOut, certRes.Certificate, 0600)
	if err != nil {
		logger().Fatalf("Unable to save Certificate for domain %s\n\t%s", certRes.Domain, err.Error())
	}

	err = ioutil.WriteFile(privOut, certRes.PrivateKey, 0600)
	if err != nil {
		logger().Fatalf("Unable to save PrivateKey for domain %s\n\t%s", certRes.Domain, err.Error())
	}

	jsonBytes, err := json.MarshalIndent(certRes, "", "\t")
	if err != nil {
		logger().Fatalf("Unable to marshal CertResource for domain %s\n\t%s", certRes.Domain, err.Error())
	}

	err = ioutil.WriteFile(metaOut, jsonBytes, 0600)
	if err != nil {
		logger().Fatalf("Unable to save CertResource for domain %s\n\t%s", certRes.Domain, err.Error())
	}
}

func run(c *cli.Context) {
	conf, acc, client := setup(c)
	if acc.Registration == nil {
		reg, err := client.Register()
		if err != nil {
			logger().Fatalf("Could not complete registration\n\t%s", err.Error())
		}

		acc.Registration = reg
		acc.Save()

		logger().Print("!!!! HEADS UP !!!!")
		logger().Printf(`
		Your account credentials have been saved in your Let's Encrypt
		configuration directory at "%s".
		You should make a secure backup	of this folder now. This
		configuration directory will also contain certificates and
		private keys obtained from Let's Encrypt so making regular
		backups of this folder is ideal.`, conf.AccountPath(c.GlobalString("email")))

	}

	if acc.Registration.Body.Agreement == "" {
		reader := bufio.NewReader(os.Stdin)
		logger().Printf("Please review the TOS at %s", acc.Registration.TosURL)

		for {
			logger().Println("Do you accept the TOS? Y/n")
			text, err := reader.ReadString('\n')
			if err != nil {
				logger().Fatalf("Could not read from console -> %s", err.Error())
			}

			text = strings.Trim(text, "\r\n")

			if text == "n" {
				logger().Fatal("You did not accept the TOS. Unable to proceed.")
			}

			if text == "Y" || text == "y" || text == "" {
				err = client.AgreeToTOS()
				if err != nil {
					logger().Fatalf("Could not agree to tos -> %s", err)
				}
				acc.Save()
				break
			}

			logger().Println("Your input was invalid. Please answer with one of Y/y, n or by pressing enter.")
		}
	}

	if len(c.GlobalStringSlice("domains")) == 0 {
		logger().Fatal("Please specify --domains or -d")
	}

	cert, failures := client.ObtainCertificate(c.GlobalStringSlice("domains"), true, nil)
	if len(failures) > 0 {
		for k, v := range failures {
			logger().Printf("[%s] Could not obtain certificates\n\t%s", k, v.Error())
		}

		// Make sure to return a non-zero exit code if ObtainSANCertificate
		// returned at least one error. Due to us not returning partial
		// certificate we can just exit here instead of at the end.
		os.Exit(1)
	}

	err := checkFolder(conf.CertPath())
	if err != nil {
		logger().Fatalf("Cound not check/create path: %s", err.Error())
	}

	saveCertRes(cert, conf)
}

func revoke(c *cli.Context) {

	conf, _, client := setup(c)

	err := checkFolder(conf.CertPath())
	if err != nil {
		logger().Fatalf("Cound not check/create path: %s", err.Error())
	}

	for _, domain := range c.GlobalStringSlice("domains") {
		logger().Printf("Trying to revoke certificate for domain %s", domain)

		certPath := path.Join(conf.CertPath(), domain+".crt")
		certBytes, err := ioutil.ReadFile(certPath)

		err = client.RevokeCertificate(certBytes)
		if err != nil {
			logger().Fatalf("Error while revoking the certificate for domain %s\n\t%s", domain, err.Error())
		} else {
			logger().Print("Certificate was revoked.")
		}
	}
}

func renew(c *cli.Context) {
	conf, _, client := setup(c)

	if len(c.GlobalStringSlice("domains")) <= 0 {
		logger().Fatal("Please specify at least one domain.")
	}

	domain := c.GlobalStringSlice("domains")[0]

	// load the cert resource from files.
	// We store the certificate, private key and metadata in different files
	// as web servers would not be able to work with a combined file.
	certPath := path.Join(conf.CertPath(), domain+".crt")
	privPath := path.Join(conf.CertPath(), domain+".key")
	metaPath := path.Join(conf.CertPath(), domain+".json")

	certBytes, err := ioutil.ReadFile(certPath)
	if err != nil {
		logger().Fatalf("Error while loading the certificate for domain %s\n\t%s", domain, err.Error())
	}

	if c.IsSet("days") {
		expTime, err := acme.GetPEMCertExpiration(certBytes)
		if err != nil {
			logger().Printf("Could not get Certification expiration for domain %s", domain)
		}

		if int(expTime.Sub(time.Now()).Hours()/24.0) > c.Int("days") {
			return
		}
	}

	metaBytes, err := ioutil.ReadFile(metaPath)
	if err != nil {
		logger().Fatalf("Error while loading the meta data for domain %s\n\t%s", domain, err.Error())
	}

	var certRes acme.CertificateResource
	err = json.Unmarshal(metaBytes, &certRes)
	if err != nil {
		logger().Fatalf("Error while marshalling the meta data for domain %s\n\t%s", domain, err.Error())
	}

	if c.Bool("reuse-key") {
		keyBytes, err := ioutil.ReadFile(privPath)
		if err != nil {
			logger().Fatalf("Error while loading the private key for domain %s\n\t%s", domain, err.Error())
		}
		certRes.PrivateKey = keyBytes
	}

	certRes.Certificate = certBytes

	newCert, err := client.RenewCertificate(certRes, true)
	if err != nil {
		logger().Fatalf("%s", err.Error())
	}

	saveCertRes(newCert, conf)
}
<commit_msg>--dns=foo means we specifically intend to fulfill a DNS challenge<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"time"

	"github.com/codegangsta/cli"
	"github.com/xenolf/lego/acme"
)

func checkFolder(path string) error {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		return os.MkdirAll(path, 0700)
	}
	return nil
}

func setup(c *cli.Context) (*Configuration, *Account, *acme.Client) {
	err := checkFolder(c.GlobalString("path"))
	if err != nil {
		logger().Fatalf("Cound not check/create path: %s", err.Error())
	}

	conf := NewConfiguration(c)
	if len(c.GlobalString("email")) == 0 {
		logger().Fatal("You have to pass an account (email address) to the program using --email or -m")
	}

	//TODO: move to account struct? Currently MUST pass email.
	acc := NewAccount(c.GlobalString("email"), conf)

	client, err := acme.NewClient(c.GlobalString("server"), acc, conf.RsaBits())
	if err != nil {
		logger().Fatalf("Could not create client: %s", err.Error())
	}

	if len(c.GlobalStringSlice("exclude")) > 0 {
		client.ExcludeChallenges(conf.ExcludedSolvers())
	}

	if c.GlobalIsSet("http") {
		client.SetHTTPAddress(c.GlobalString("http"))
	}

	if c.GlobalIsSet("tls") {
		client.SetTLSAddress(c.GlobalString("tls"))
	}

	if c.GlobalIsSet("dns") {
		var err error
		var provider acme.ChallengeProvider
		switch c.GlobalString("dns") {
		case "cloudflare":
			provider, err = acme.NewDNSProviderCloudFlare("", "")
		case "digitalocean":
			authToken := os.Getenv("DO_AUTH_TOKEN")

			provider, err = acme.NewDNSProviderDigitalOcean(authToken)
		case "dnsimple":
			provider, err = acme.NewDNSProviderDNSimple("", "")
		case "route53":
			awsRegion := os.Getenv("AWS_REGION")
			provider, err = acme.NewDNSProviderRoute53("", "", awsRegion)
		case "rfc2136":
			nameserver := os.Getenv("RFC2136_NAMESERVER")
			zone := os.Getenv("RFC2136_ZONE")
			tsigAlgorithm := os.Getenv("RFC2136_TSIG_ALGORITHM")
			tsigKey := os.Getenv("RFC2136_TSIG_KEY")
			tsigSecret := os.Getenv("RFC2136_TSIG_SECRET")

			provider, err = acme.NewDNSProviderRFC2136(nameserver, zone, tsigAlgorithm, tsigKey, tsigSecret)
		case "manual":
			provider, err = acme.NewDNSProviderManual()
		}

		if err != nil {
			logger().Fatal(err)
		}

		client.SetChallengeProvider(acme.DNS01, provider)

		// --dns=foo indicates that the user specifically want to do a DNS challenge
		// infer that the user also wants to exclude all other challenges
		client.ExcludeChallenges([]acme.Challenge{acme.HTTP01, acme.TLSSNI01})
	}

	return conf, acc, client
}

func saveCertRes(certRes acme.CertificateResource, conf *Configuration) {
	// We store the certificate, private key and metadata in different files
	// as web servers would not be able to work with a combined file.
	certOut := path.Join(conf.CertPath(), certRes.Domain+".crt")
	privOut := path.Join(conf.CertPath(), certRes.Domain+".key")
	metaOut := path.Join(conf.CertPath(), certRes.Domain+".json")

	err := ioutil.WriteFile(certOut, certRes.Certificate, 0600)
	if err != nil {
		logger().Fatalf("Unable to save Certificate for domain %s\n\t%s", certRes.Domain, err.Error())
	}

	err = ioutil.WriteFile(privOut, certRes.PrivateKey, 0600)
	if err != nil {
		logger().Fatalf("Unable to save PrivateKey for domain %s\n\t%s", certRes.Domain, err.Error())
	}

	jsonBytes, err := json.MarshalIndent(certRes, "", "\t")
	if err != nil {
		logger().Fatalf("Unable to marshal CertResource for domain %s\n\t%s", certRes.Domain, err.Error())
	}

	err = ioutil.WriteFile(metaOut, jsonBytes, 0600)
	if err != nil {
		logger().Fatalf("Unable to save CertResource for domain %s\n\t%s", certRes.Domain, err.Error())
	}
}

func run(c *cli.Context) {
	conf, acc, client := setup(c)
	if acc.Registration == nil {
		reg, err := client.Register()
		if err != nil {
			logger().Fatalf("Could not complete registration\n\t%s", err.Error())
		}

		acc.Registration = reg
		acc.Save()

		logger().Print("!!!! HEADS UP !!!!")
		logger().Printf(`
		Your account credentials have been saved in your Let's Encrypt
		configuration directory at "%s".
		You should make a secure backup	of this folder now. This
		configuration directory will also contain certificates and
		private keys obtained from Let's Encrypt so making regular
		backups of this folder is ideal.`, conf.AccountPath(c.GlobalString("email")))

	}

	if acc.Registration.Body.Agreement == "" {
		reader := bufio.NewReader(os.Stdin)
		logger().Printf("Please review the TOS at %s", acc.Registration.TosURL)

		for {
			logger().Println("Do you accept the TOS? Y/n")
			text, err := reader.ReadString('\n')
			if err != nil {
				logger().Fatalf("Could not read from console -> %s", err.Error())
			}

			text = strings.Trim(text, "\r\n")

			if text == "n" {
				logger().Fatal("You did not accept the TOS. Unable to proceed.")
			}

			if text == "Y" || text == "y" || text == "" {
				err = client.AgreeToTOS()
				if err != nil {
					logger().Fatalf("Could not agree to tos -> %s", err)
				}
				acc.Save()
				break
			}

			logger().Println("Your input was invalid. Please answer with one of Y/y, n or by pressing enter.")
		}
	}

	if len(c.GlobalStringSlice("domains")) == 0 {
		logger().Fatal("Please specify --domains or -d")
	}

	cert, failures := client.ObtainCertificate(c.GlobalStringSlice("domains"), true, nil)
	if len(failures) > 0 {
		for k, v := range failures {
			logger().Printf("[%s] Could not obtain certificates\n\t%s", k, v.Error())
		}

		// Make sure to return a non-zero exit code if ObtainSANCertificate
		// returned at least one error. Due to us not returning partial
		// certificate we can just exit here instead of at the end.
		os.Exit(1)
	}

	err := checkFolder(conf.CertPath())
	if err != nil {
		logger().Fatalf("Cound not check/create path: %s", err.Error())
	}

	saveCertRes(cert, conf)
}

func revoke(c *cli.Context) {

	conf, _, client := setup(c)

	err := checkFolder(conf.CertPath())
	if err != nil {
		logger().Fatalf("Cound not check/create path: %s", err.Error())
	}

	for _, domain := range c.GlobalStringSlice("domains") {
		logger().Printf("Trying to revoke certificate for domain %s", domain)

		certPath := path.Join(conf.CertPath(), domain+".crt")
		certBytes, err := ioutil.ReadFile(certPath)

		err = client.RevokeCertificate(certBytes)
		if err != nil {
			logger().Fatalf("Error while revoking the certificate for domain %s\n\t%s", domain, err.Error())
		} else {
			logger().Print("Certificate was revoked.")
		}
	}
}

func renew(c *cli.Context) {
	conf, _, client := setup(c)

	if len(c.GlobalStringSlice("domains")) <= 0 {
		logger().Fatal("Please specify at least one domain.")
	}

	domain := c.GlobalStringSlice("domains")[0]

	// load the cert resource from files.
	// We store the certificate, private key and metadata in different files
	// as web servers would not be able to work with a combined file.
	certPath := path.Join(conf.CertPath(), domain+".crt")
	privPath := path.Join(conf.CertPath(), domain+".key")
	metaPath := path.Join(conf.CertPath(), domain+".json")

	certBytes, err := ioutil.ReadFile(certPath)
	if err != nil {
		logger().Fatalf("Error while loading the certificate for domain %s\n\t%s", domain, err.Error())
	}

	if c.IsSet("days") {
		expTime, err := acme.GetPEMCertExpiration(certBytes)
		if err != nil {
			logger().Printf("Could not get Certification expiration for domain %s", domain)
		}

		if int(expTime.Sub(time.Now()).Hours()/24.0) > c.Int("days") {
			return
		}
	}

	metaBytes, err := ioutil.ReadFile(metaPath)
	if err != nil {
		logger().Fatalf("Error while loading the meta data for domain %s\n\t%s", domain, err.Error())
	}

	var certRes acme.CertificateResource
	err = json.Unmarshal(metaBytes, &certRes)
	if err != nil {
		logger().Fatalf("Error while marshalling the meta data for domain %s\n\t%s", domain, err.Error())
	}

	if c.Bool("reuse-key") {
		keyBytes, err := ioutil.ReadFile(privPath)
		if err != nil {
			logger().Fatalf("Error while loading the private key for domain %s\n\t%s", domain, err.Error())
		}
		certRes.PrivateKey = keyBytes
	}

	certRes.Certificate = certBytes

	newCert, err := client.RenewCertificate(certRes, true)
	if err != nil {
		logger().Fatalf("%s", err.Error())
	}

	saveCertRes(newCert, conf)
}
<|endoftext|>


<commit_before>// Copyright 2015 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"errors"
	"path"
	"time"

	"github.com/control-center/serviced/coordinator/client"
	"github.com/control-center/serviced/domain/service"
	ss "github.com/control-center/serviced/domain/servicestate"
	"github.com/zenoss/glog"
)

const (
	// zkServiceAlert alerts services when instances are added or deleted.
	zkServiceAlert = "/alerts/services"
	// zkInstanceLock keeps service instance updates in sync.
	zkInstanceLock = "/locks/instances"
)

const (
	// AlertInitialized indicates that the alerter has been initialized for the
	// service.
	AlertInitialized = "INIT"
	// InstanceAdded describes a service event alert for an instance that was
	// created.
	InstanceAdded = "ADD"
	// InstanceDeleted describes a service event alert for an instance that was
	// deleted.
	InstanceDeleted = "DEL"
)

var ErrLockNotFound = errors.New("lock not found")

// ServiceAlert is a alert node for when a service instance is added or
// deleted.
type ServiceAlert struct {
	ServiceID string
	HostID    string
	StateID   string
	Event     string
	Timestamp time.Time
	version   interface{}
}

// Version implements client.Node
func (alert *ServiceAlert) Version() interface{} {
	return alert.version
}

// SetVersion implements client.Node
func (alert *ServiceAlert) SetVersion(version interface{}) {
	alert.version = version
}

// setUpAlert sets up the service alerter.
func setupAlert(conn client.Connection, serviceID string) error {
	var alert ServiceAlert
	if err := conn.Create(path.Join(zkServiceAlert, serviceID), &alert); err == nil {
		alert.Event = AlertInitialized
		alert.Timestamp = time.Now()
		if err := conn.Set(path.Join(zkServiceAlert, serviceID), &alert); err != nil {
			glog.Errorf("Could not set alerter for service %s: %s", serviceID, err)
			return err
		}
	} else if err != client.ErrNodeExists {
		glog.Errorf("Could not create alerter for service %s: %s", serviceID, err)
		return err
	}
	return nil
}

// removeAlert cleans up service alerter.
func removeAlert(conn client.Connection, serviceID string) error {
	return conn.Delete(path.Join(zkServiceAlert, serviceID))
}

// alertService sends a notification to a service that one of its service
// instances has been updated.  And will set the value of the last updated
// instance.
func alertService(conn client.Connection, serviceID, hostID, stateID, event string) error {
	var alert ServiceAlert
	if err := conn.Get(path.Join(zkServiceAlert, serviceID), &alert); err != nil && err != client.ErrEmptyNode {
		glog.Errorf("Could not find service %s: %s", serviceID, err)
		return err
	}
	alert.ServiceID = serviceID
	alert.HostID = hostID
	alert.StateID = stateID
	alert.Event = event
	alert.Timestamp = time.Now()
	if err := conn.Set(path.Join(zkServiceAlert, serviceID), &alert); err != nil {
		glog.Errorf("Could not alert service %s: %s", serviceID, err)
		return err
	}
	return nil
}

// newInstanceLock sets up a new zk instance lock for a given service state id
func newInstanceLock(conn client.Connection, stateID string) client.Lock {
	return conn.NewLock(path.Join(zkInstanceLock, stateID))
}

// rmInstanceLock removes a zk instance lock parent
func rmInstanceLock(conn client.Connection, stateID string) error {
	return conn.Delete(path.Join(zkInstanceLock, stateID))
}

// addInstance creates a new service state and host instance
func addInstance(conn client.Connection, state ss.ServiceState) error {
	glog.V(2).Infof("Adding instance %+v", state)
	// check the object
	if err := state.ValidEntity(); err != nil {
		glog.Errorf("Could not validate service state %+v: %s", state, err)
		return err
	}

	lock := newInstanceLock(conn, state.ID)
	if err := lock.Lock(); err != nil {
		glog.Errorf("Could not set lock for service instance %s for service %s on host %s: %s", state.ID, state.ServiceID, state.HostID, err)
		return err
	}
	glog.V(2).Infof("Acquired lock for instance %s", state.ID)
	defer lock.Unlock()
	defer alertService(conn, state.ServiceID, state.HostID, state.ID, InstanceAdded)

	var err error
	defer func() {
		if err != nil {
			conn.Delete(hostpath(state.HostID, state.ID))
			conn.Delete(servicepath(state.ServiceID, state.ID))
			rmInstanceLock(conn, state.ID)
		}
	}()

	// Create node on the service
	spath := servicepath(state.ServiceID, state.ID)
	snode := &ServiceStateNode{ServiceState: &state}
	if err = conn.Create(spath, snode); err != nil {
		glog.Errorf("Could not create service state %s for service %s: %s", state.ID, state.ServiceID, err)
		return err
	} else if err = conn.Set(spath, snode); err != nil {
		glog.Errorf("Could not set service state %s for node %+v: %s", state.ID, snode, err)
		return err
	}

	// Create node on the host
	hpath := hostpath(state.HostID, state.ID)
	hnode := NewHostState(&state)
	glog.V(2).Infof("Host node: %+v", hnode)
	if err = conn.Create(hpath, hnode); err != nil {
		glog.Errorf("Could not create host state %s for host %s: %s", state.ID, state.HostID, err)
		return err
	} else if err = conn.Set(hpath, hnode); err != nil {
		glog.Errorf("Could not set host state %s for node %+v: %s", state.ID, hnode, err)
		return err
	}

	glog.V(2).Infof("Releasing lock for instance %s", state.ID)
	return nil
}

// removeInstance removes the service state and host instances
func removeInstance(conn client.Connection, serviceID, hostID, stateID string) error {
	glog.V(2).Infof("Removing instance %s", stateID)

	lock := newInstanceLock(conn, stateID)
	if err := lock.Lock(); err != nil {
		glog.Errorf("Could not set lock for service instance %s for service %s on host %s: %s", stateID, serviceID, hostID, err)
		return err
	}
	defer lock.Unlock()
	defer alertService(conn, serviceID, hostID, stateID, InstanceDeleted)
	defer rmInstanceLock(conn, stateID)
	glog.V(2).Infof("Acquired lock for instance %s", stateID)
	// Remove the node on the service
	spath := servicepath(serviceID, stateID)
	if err := conn.Delete(spath); err != nil {
		if err != client.ErrNoNode {
			glog.Errorf("Could not delete service state node %s for service %s on host %s: %s", stateID, serviceID, hostID, err)
			return err
		}
	}
	// Remove the node on the host
	hpath := hostpath(hostID, stateID)
	if err := conn.Delete(hpath); err != nil {
		if err != client.ErrNoNode {
			glog.Errorf("Could not delete host state node %s for host %s: %s", stateID, hostID, err)
			return err
		}
	}
	glog.V(2).Infof("Releasing lock for instance %s", stateID)
	return nil
}

// updateInstance updates the service state and host instances
func updateInstance(conn client.Connection, hostID, stateID string, mutate func(*HostState, *ss.ServiceState)) error {
	glog.V(2).Infof("Updating instance %s", stateID)
	// do not lock if parent lock does not exist
	if exists, err := conn.Exists(path.Join(zkInstanceLock, stateID)); err != nil && err != client.ErrNoNode {
		glog.Errorf("Could not check for lock on instance %s: %s", stateID, err)
		return err
	} else if !exists {
		glog.Errorf("Lock not found for instance %s", stateID)
		return ErrLockNotFound
	}

	lock := newInstanceLock(conn, stateID)
	if err := lock.Lock(); err != nil {
		glog.Errorf("Could not set lock for service instance %s on host %s: %s", stateID, hostID, err)
		return err
	}
	defer lock.Unlock()
	glog.V(2).Infof("Acquired lock for instance %s", stateID)

	hpath := hostpath(hostID, stateID)
	var hsdata HostState
	if err := conn.Get(hpath, &hsdata); err != nil {
		glog.Errorf("Could not get instance %s for host %s: %s", stateID, hostID, err)
		return err
	}
	serviceID := hsdata.ServiceID
	spath := servicepath(serviceID, stateID)
	var ssnode ServiceStateNode
	if err := conn.Get(spath, &ssnode); err != nil {
		glog.Errorf("Could not get instance %s for service %s: %s", stateID, serviceID, err)
		return err
	}

	mutate(&hsdata, ssnode.ServiceState)

	if err := conn.Set(hpath, &hsdata); err != nil {
		glog.Errorf("Could not update instance %s for host %s: %s", stateID, hostID, err)
		return err
	}
	if err := conn.Set(spath, &ssnode); err != nil {
		glog.Errorf("Could not update instance %s for service %s: %s", stateID, serviceID, err)
		return err
	}
	glog.V(2).Infof("Releasing lock for instance %s", stateID)
	return nil
}

// removeInstancesOnHost removes all instances for a particular host. Will not
// delete if the instance cannot be found on the host (for when you have
// incongruent data).
func removeInstancesOnHost(conn client.Connection, hostID string) {
	instances, err := conn.Children(hostpath(hostID))
	if err != nil {
		glog.Errorf("Could not look up instances on host %s: %s", hostID, err)
		return
	}
	for _, stateID := range instances {
		var hs HostState
		if err := conn.Get(hostpath(hostID, stateID), &hs); err != nil {
			glog.Warningf("Could not look up host instance %s on host %s: %s", stateID, hostID, err)
		} else if err := removeInstance(conn, hs.ServiceID, hs.HostID, hs.ServiceStateID); err != nil {
			glog.Warningf("Could not remove host instance %s on host %s for service %s: %s", hs.ServiceStateID, hs.HostID, hs.ServiceID, err)
		} else {
			glog.V(2).Infof("Removed instance %s on host %s for service %s", hs.ServiceStateID, hs.HostID, hs.ServiceID, err)
		}
	}
}

// removeInstancesOnService removes all instances for a particular service. Will
// not delete if the instance cannot be found on the service (for when you have
// incongruent data).
func removeInstancesOnService(conn client.Connection, serviceID string) {
	instances, err := conn.Children(servicepath(serviceID))
	if err != nil {
		glog.Errorf("Could not look up instances on service %s: %s", serviceID, err)
		return
	}
	for _, stateID := range instances {
		var state ss.ServiceState
		if err := conn.Get(servicepath(serviceID, stateID), &ServiceStateNode{ServiceState: &state}); err != nil {
			glog.Warningf("Could not look up service instance %s for service %s: %s", stateID, serviceID, err)
		} else if err := removeInstance(conn, state.ServiceID, state.HostID, state.ID); err != nil {
			glog.Warningf("Could not remove service instance %s for service %s on host %s: %s", state.ID, state.ServiceID, state.HostID, err)
		} else {
			glog.V(2).Infof("Removed instance %s for service %s on host %s", state.ID, state.ServiceID, state.HostID, err)
		}
	}
}

// pauseInstance pauses a running host state instance
func pauseInstance(conn client.Connection, hostID, stateID string) error {
	return updateInstance(conn, hostID, stateID, func(hsdata *HostState, _ *ss.ServiceState) {
		if hsdata.DesiredState == int(service.SVCRun) {
			glog.V(2).Infof("Pausing service instance %s via host %s", stateID, hostID)
			hsdata.DesiredState = int(service.SVCPause)
		}
	})
}

// resumeInstance resumes a paused host state instance
func resumeInstance(conn client.Connection, hostID, stateID string) error {
	return updateInstance(conn, hostID, stateID, func(hsdata *HostState, _ *ss.ServiceState) {
		if hsdata.DesiredState == int(service.SVCPause) {
			glog.V(2).Infof("Resuming service instance %s via host %s", stateID, hostID)
			hsdata.DesiredState = int(service.SVCRun)
		}
	})
}

// UpdateServiceState does a full update of a service state
func UpdateServiceState(conn client.Connection, state *ss.ServiceState) error {
	if err := state.ValidEntity(); err != nil {
		glog.Errorf("Could not validate service state %+v: %s", state, err)
		return err
	}
	return updateInstance(conn, state.HostID, state.ID, func(_ *HostState, ssdata *ss.ServiceState) {
		*ssdata = *state
	})
}

// StopServiceInstance stops a host state instance
func StopServiceInstance(conn client.Connection, hostID, stateID string) error {
	return updateInstance(conn, hostID, stateID, func(hsdata *HostState, _ *ss.ServiceState) {
		glog.V(2).Infof("Stopping service instance via %s host %s", stateID, hostID)
		hsdata.DesiredState = int(service.SVCStop)
	})
}<commit_msg>Check for lock existence before attempting lock during instance removal.<commit_after>// Copyright 2015 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"errors"
	"path"
	"time"

	"github.com/control-center/serviced/coordinator/client"
	"github.com/control-center/serviced/domain/service"
	ss "github.com/control-center/serviced/domain/servicestate"
	"github.com/zenoss/glog"
)

const (
	// zkServiceAlert alerts services when instances are added or deleted.
	zkServiceAlert = "/alerts/services"
	// zkInstanceLock keeps service instance updates in sync.
	zkInstanceLock = "/locks/instances"
)

const (
	// AlertInitialized indicates that the alerter has been initialized for the
	// service.
	AlertInitialized = "INIT"
	// InstanceAdded describes a service event alert for an instance that was
	// created.
	InstanceAdded = "ADD"
	// InstanceDeleted describes a service event alert for an instance that was
	// deleted.
	InstanceDeleted = "DEL"
)

var ErrLockNotFound = errors.New("lock not found")

// ServiceAlert is a alert node for when a service instance is added or
// deleted.
type ServiceAlert struct {
	ServiceID string
	HostID    string
	StateID   string
	Event     string
	Timestamp time.Time
	version   interface{}
}

// Version implements client.Node
func (alert *ServiceAlert) Version() interface{} {
	return alert.version
}

// SetVersion implements client.Node
func (alert *ServiceAlert) SetVersion(version interface{}) {
	alert.version = version
}

// setUpAlert sets up the service alerter.
func setupAlert(conn client.Connection, serviceID string) error {
	var alert ServiceAlert
	if err := conn.Create(path.Join(zkServiceAlert, serviceID), &alert); err == nil {
		alert.Event = AlertInitialized
		alert.Timestamp = time.Now()
		if err := conn.Set(path.Join(zkServiceAlert, serviceID), &alert); err != nil {
			glog.Errorf("Could not set alerter for service %s: %s", serviceID, err)
			return err
		}
	} else if err != client.ErrNodeExists {
		glog.Errorf("Could not create alerter for service %s: %s", serviceID, err)
		return err
	}
	return nil
}

// removeAlert cleans up service alerter.
func removeAlert(conn client.Connection, serviceID string) error {
	return conn.Delete(path.Join(zkServiceAlert, serviceID))
}

// alertService sends a notification to a service that one of its service
// instances has been updated.  And will set the value of the last updated
// instance.
func alertService(conn client.Connection, serviceID, hostID, stateID, event string) error {
	var alert ServiceAlert
	if err := conn.Get(path.Join(zkServiceAlert, serviceID), &alert); err != nil && err != client.ErrEmptyNode {
		glog.Errorf("Could not find service %s: %s", serviceID, err)
		return err
	}
	alert.ServiceID = serviceID
	alert.HostID = hostID
	alert.StateID = stateID
	alert.Event = event
	alert.Timestamp = time.Now()
	if err := conn.Set(path.Join(zkServiceAlert, serviceID), &alert); err != nil {
		glog.Errorf("Could not alert service %s: %s", serviceID, err)
		return err
	}
	return nil
}

// newInstanceLock sets up a new zk instance lock for a given service state id
func newInstanceLock(conn client.Connection, stateID string) client.Lock {
	return conn.NewLock(path.Join(zkInstanceLock, stateID))
}

// rmInstanceLock removes a zk instance lock parent
func rmInstanceLock(conn client.Connection, stateID string) error {
	return conn.Delete(path.Join(zkInstanceLock, stateID))
}

// addInstance creates a new service state and host instance
func addInstance(conn client.Connection, state ss.ServiceState) error {
	glog.V(2).Infof("Adding instance %+v", state)
	// check the object
	if err := state.ValidEntity(); err != nil {
		glog.Errorf("Could not validate service state %+v: %s", state, err)
		return err
	}

	lock := newInstanceLock(conn, state.ID)
	if err := lock.Lock(); err != nil {
		glog.Errorf("Could not set lock for service instance %s for service %s on host %s: %s", state.ID, state.ServiceID, state.HostID, err)
		return err
	}
	glog.V(2).Infof("Acquired lock for instance %s", state.ID)
	defer lock.Unlock()
	defer alertService(conn, state.ServiceID, state.HostID, state.ID, InstanceAdded)

	var err error
	defer func() {
		if err != nil {
			conn.Delete(hostpath(state.HostID, state.ID))
			conn.Delete(servicepath(state.ServiceID, state.ID))
			rmInstanceLock(conn, state.ID)
		}
	}()

	// Create node on the service
	spath := servicepath(state.ServiceID, state.ID)
	snode := &ServiceStateNode{ServiceState: &state}
	if err = conn.Create(spath, snode); err != nil {
		glog.Errorf("Could not create service state %s for service %s: %s", state.ID, state.ServiceID, err)
		return err
	} else if err = conn.Set(spath, snode); err != nil {
		glog.Errorf("Could not set service state %s for node %+v: %s", state.ID, snode, err)
		return err
	}

	// Create node on the host
	hpath := hostpath(state.HostID, state.ID)
	hnode := NewHostState(&state)
	glog.V(2).Infof("Host node: %+v", hnode)
	if err = conn.Create(hpath, hnode); err != nil {
		glog.Errorf("Could not create host state %s for host %s: %s", state.ID, state.HostID, err)
		return err
	} else if err = conn.Set(hpath, hnode); err != nil {
		glog.Errorf("Could not set host state %s for node %+v: %s", state.ID, hnode, err)
		return err
	}

	glog.V(2).Infof("Releasing lock for instance %s", state.ID)
	return nil
}

// removeInstance removes the service state and host instances
func removeInstance(conn client.Connection, serviceID, hostID, stateID string) error {
	glog.V(2).Infof("Removing instance %s", stateID)

	if exists, err := conn.Exists(path.Join(zkInstanceLock, stateID)); err != nil && err != client.ErrNoNode {
		glog.Errorf("Could not check for lock on instance %s: %s", stateID, err)
		return err
	} else if exists {
		lock := newInstanceLock(conn, stateID)
		if err := lock.Lock(); err != nil {
			glog.Errorf("Could not set lock for service instance %s for service %s on host %s: %s", stateID, serviceID, hostID, err)
			return err
		}
		defer lock.Unlock()
		defer alertService(conn, serviceID, hostID, stateID, InstanceDeleted)
		defer rmInstanceLock(conn, stateID)
		glog.V(2).Infof("Acquired lock for instance %s", stateID)
	}

	// Remove the node on the service
	spath := servicepath(serviceID, stateID)
	if err := conn.Delete(spath); err != nil {
		if err != client.ErrNoNode {
			glog.Errorf("Could not delete service state node %s for service %s on host %s: %s", stateID, serviceID, hostID, err)
			return err
		}
	}
	// Remove the node on the host
	hpath := hostpath(hostID, stateID)
	if err := conn.Delete(hpath); err != nil {
		if err != client.ErrNoNode {
			glog.Errorf("Could not delete host state node %s for host %s: %s", stateID, hostID, err)
			return err
		}
	}
	glog.V(2).Infof("Releasing lock for instance %s", stateID)
	return nil
}

// updateInstance updates the service state and host instances
func updateInstance(conn client.Connection, hostID, stateID string, mutate func(*HostState, *ss.ServiceState)) error {
	glog.V(2).Infof("Updating instance %s", stateID)
	// do not lock if parent lock does not exist
	if exists, err := conn.Exists(path.Join(zkInstanceLock, stateID)); err != nil && err != client.ErrNoNode {
		glog.Errorf("Could not check for lock on instance %s: %s", stateID, err)
		return err
	} else if !exists {
		glog.Errorf("Lock not found for instance %s", stateID)
		return ErrLockNotFound
	}

	lock := newInstanceLock(conn, stateID)
	if err := lock.Lock(); err != nil {
		glog.Errorf("Could not set lock for service instance %s on host %s: %s", stateID, hostID, err)
		return err
	}
	defer lock.Unlock()
	glog.V(2).Infof("Acquired lock for instance %s", stateID)

	hpath := hostpath(hostID, stateID)
	var hsdata HostState
	if err := conn.Get(hpath, &hsdata); err != nil {
		glog.Errorf("Could not get instance %s for host %s: %s", stateID, hostID, err)
		return err
	}
	serviceID := hsdata.ServiceID
	spath := servicepath(serviceID, stateID)
	var ssnode ServiceStateNode
	if err := conn.Get(spath, &ssnode); err != nil {
		glog.Errorf("Could not get instance %s for service %s: %s", stateID, serviceID, err)
		return err
	}

	mutate(&hsdata, ssnode.ServiceState)

	if err := conn.Set(hpath, &hsdata); err != nil {
		glog.Errorf("Could not update instance %s for host %s: %s", stateID, hostID, err)
		return err
	}
	if err := conn.Set(spath, &ssnode); err != nil {
		glog.Errorf("Could not update instance %s for service %s: %s", stateID, serviceID, err)
		return err
	}
	glog.V(2).Infof("Releasing lock for instance %s", stateID)
	return nil
}

// removeInstancesOnHost removes all instances for a particular host. Will not
// delete if the instance cannot be found on the host (for when you have
// incongruent data).
func removeInstancesOnHost(conn client.Connection, hostID string) {
	instances, err := conn.Children(hostpath(hostID))
	if err != nil {
		glog.Errorf("Could not look up instances on host %s: %s", hostID, err)
		return
	}
	for _, stateID := range instances {
		var hs HostState
		if err := conn.Get(hostpath(hostID, stateID), &hs); err != nil {
			glog.Warningf("Could not look up host instance %s on host %s: %s", stateID, hostID, err)
		} else if err := removeInstance(conn, hs.ServiceID, hs.HostID, hs.ServiceStateID); err != nil {
			glog.Warningf("Could not remove host instance %s on host %s for service %s: %s", hs.ServiceStateID, hs.HostID, hs.ServiceID, err)
		} else {
			glog.V(2).Infof("Removed instance %s on host %s for service %s", hs.ServiceStateID, hs.HostID, hs.ServiceID, err)
		}
	}
}

// removeInstancesOnService removes all instances for a particular service. Will
// not delete if the instance cannot be found on the service (for when you have
// incongruent data).
func removeInstancesOnService(conn client.Connection, serviceID string) {
	instances, err := conn.Children(servicepath(serviceID))
	if err != nil {
		glog.Errorf("Could not look up instances on service %s: %s", serviceID, err)
		return
	}
	for _, stateID := range instances {
		var state ss.ServiceState
		if err := conn.Get(servicepath(serviceID, stateID), &ServiceStateNode{ServiceState: &state}); err != nil {
			glog.Warningf("Could not look up service instance %s for service %s: %s", stateID, serviceID, err)
		} else if err := removeInstance(conn, state.ServiceID, state.HostID, state.ID); err != nil {
			glog.Warningf("Could not remove service instance %s for service %s on host %s: %s", state.ID, state.ServiceID, state.HostID, err)
		} else {
			glog.V(2).Infof("Removed instance %s for service %s on host %s", state.ID, state.ServiceID, state.HostID, err)
		}
	}
}

// pauseInstance pauses a running host state instance
func pauseInstance(conn client.Connection, hostID, stateID string) error {
	return updateInstance(conn, hostID, stateID, func(hsdata *HostState, _ *ss.ServiceState) {
		if hsdata.DesiredState == int(service.SVCRun) {
			glog.V(2).Infof("Pausing service instance %s via host %s", stateID, hostID)
			hsdata.DesiredState = int(service.SVCPause)
		}
	})
}

// resumeInstance resumes a paused host state instance
func resumeInstance(conn client.Connection, hostID, stateID string) error {
	return updateInstance(conn, hostID, stateID, func(hsdata *HostState, _ *ss.ServiceState) {
		if hsdata.DesiredState == int(service.SVCPause) {
			glog.V(2).Infof("Resuming service instance %s via host %s", stateID, hostID)
			hsdata.DesiredState = int(service.SVCRun)
		}
	})
}

// UpdateServiceState does a full update of a service state
func UpdateServiceState(conn client.Connection, state *ss.ServiceState) error {
	if err := state.ValidEntity(); err != nil {
		glog.Errorf("Could not validate service state %+v: %s", state, err)
		return err
	}
	return updateInstance(conn, state.HostID, state.ID, func(_ *HostState, ssdata *ss.ServiceState) {
		*ssdata = *state
	})
}

// StopServiceInstance stops a host state instance
func StopServiceInstance(conn client.Connection, hostID, stateID string) error {
	return updateInstance(conn, hostID, stateID, func(hsdata *HostState, _ *ss.ServiceState) {
		glog.V(2).Infof("Stopping service instance via %s host %s", stateID, hostID)
		hsdata.DesiredState = int(service.SVCStop)
	})
}<|endoftext|>