<commit_before>// SPDX-License-Identifier: ISC
// Copyright (c) 2014-2019 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package reservoir

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"os"

	"github.com/bitmark-inc/bitmarkd/asset"
	"github.com/bitmark-inc/bitmarkd/fault"
	"github.com/bitmark-inc/bitmarkd/mode"
	"github.com/bitmark-inc/bitmarkd/pay"
	"github.com/bitmark-inc/bitmarkd/storage"
	"github.com/bitmark-inc/bitmarkd/transactionrecord"
	"github.com/bitmark-inc/logger"
)

type tagType byte

// record types in cache file
const (
	taggedBOF         tagType = iota
	taggedEOF         tagType = iota
	taggedTransaction tagType = iota
	taggedProof       tagType = iota
)

// the BOF tag to chec file version
// exact match is required
var bofData = []byte("bitmark-cache v1.0")

// Handles - storage handles used when restore from cache file
type Handles struct {
	Assets            storage.Handle
	BlockOwnerPayment storage.Handle
	Transaction       storage.Handle
	OwnerTx           storage.Handle
	OwnerData         storage.Handle
	Share             storage.Handle
	ShareQuantity     storage.Handle
}

// LoadFromFile - load transactions from file
// called later when system is able to handle the tx and proofs
func LoadFromFile(handles Handles) error {
	Disable()
	defer Enable()

	log := globalData.log

	log.Info("starting…")

	f, err := os.Open(globalData.filename)
	if nil != err {
		return err
	}
	defer f.Close()

	// must have BOF record first
	tag, packed, err := readRecord(f)
	if nil != err {
		return err
	}

	if taggedBOF != tag {
		return fmt.Errorf("expected BOF: %d but read: %d", taggedBOF, tag)
	}

	if !bytes.Equal(bofData, packed) {
		return fmt.Errorf("expected BOF: %q but read: %q", bofData, packed)
	}

	log.Infof("restore from file: %s", globalData.filename)

restore_loop:
	for {
		tag, packed, err := readRecord(f)
		if nil != err {
			log.Errorf("read record with error: %s\n", err)
			continue restore_loop
		}

		switch tag {

		case taggedEOF:
			break restore_loop

		case taggedTransaction:
			unpacked, _, err := packed.Unpack(mode.IsTesting())
			if nil != err {
				log.Errorf("unable to unpack asset: %s", err)
				continue restore_loop
			}

			restorer, err := NewTransactionRestorer(unpacked, packed, handles)
			if nil != err {
				log.Errorf("create transaction restorer with error: %s", err)
				continue restore_loop
			}

			err = restorer.Restore()
			if nil != err {
				log.Errorf("restore %s with error: %s", restorer, err)
				continue restore_loop
			}

		case taggedProof:
			var payId pay.PayId
			pn := len(payId)
			if len(packed) <= pn {
				log.Errorf("unable to unpack proof: record too short: %d  expected > %d", len(packed), pn)
				continue restore_loop
			}
			copy(payId[:], packed[:pn])
			nonce := packed[pn:]
			TryProof(payId, nonce)

		default:
			// in case any unsupported tag exist
			msg := fmt.Errorf("abort, read invalid tag: 0x%02x", tag)
			log.Error(msg.Error())
			return msg
		}
	}
	log.Info("restore completed")
	return nil
}

// save transactions to file
func saveToFile() error {
	globalData.Lock()
	defer globalData.Unlock()

	log := globalData.log

	if !globalData.initialised {
		log.Error("save when not initialised")
		return fault.NotInitialised
	}

	log.Info("saving…")

	f, err := os.OpenFile(globalData.filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
	if nil != err {
		return err
	}
	defer f.Close()

	// write beginning of file marker
	err = writeRecord(f, taggedBOF, bofData)
	if nil != err {
		return err
	}

	// all assets at start of file
	err = backupAssets(f)
	if nil != err {
		return err
	}

	// verified

	for _, item := range globalData.verifiedTransactions {
		err := writeRecord(f, taggedTransaction, item.packed)
		if nil != err {
			return err
		}
	}
	for _, item := range globalData.verifiedFreeIssues {
		err := writeBlock(f, taggedTransaction, item.txs)
		if nil != err {
			return err
		}
		err = writeRecord(f, taggedProof, packProof(item.payId, item.nonce))
		if nil != err {
			return err
		}
	}
	for _, item := range globalData.verifiedPaidIssues {
		err := writeBlock(f, taggedTransaction, item.txs)
		if nil != err {
			return err
		}
	}

	// pending

	for _, item := range globalData.pendingTransactions {
		err := writeRecord(f, taggedTransaction, item.tx.packed)
		if nil != err {
			return err
		}
	}
	for _, item := range globalData.pendingFreeIssues {
		err := writeBlock(f, taggedTransaction, item.txs)
		if nil != err {
			return err
		}
		err = writeRecord(f, taggedProof, packProof(item.payId, item.nonce))
		if nil != err {
			return err
		}
	}
	for _, item := range globalData.pendingPaidIssues {
		err := writeBlock(f, taggedTransaction, item.txs)
		if nil != err {
			return err
		}
	}

	// end the file
	err = writeRecord(f, taggedEOF, []byte("EOF"))
	if nil != err {
		return err
	}

	log.Info("save completed")
	return nil
}

func backupAssets(f *os.File) error {
	allAssets := make(map[transactionrecord.AssetIdentifier]struct{})

	// verified

	for _, item := range globalData.verifiedFreeIssues {
		for _, tx := range item.txs {
			if tx, ok := tx.transaction.(*transactionrecord.BitmarkIssue); ok {
				allAssets[tx.AssetId] = struct{}{}
			}
		}
	}
	for _, item := range globalData.verifiedPaidIssues {
		for _, tx := range item.txs {
			if tx, ok := tx.transaction.(*transactionrecord.BitmarkIssue); ok {
				allAssets[tx.AssetId] = struct{}{}
			}
		}
	}

	// pending

	for _, item := range globalData.pendingFreeIssues {
		for _, tx := range item.txs {
			if tx, ok := tx.transaction.(*transactionrecord.BitmarkIssue); ok {
				allAssets[tx.AssetId] = struct{}{}
			}
		}
	}
	for _, item := range globalData.pendingPaidIssues {
		for _, tx := range item.txs {
			if tx, ok := tx.transaction.(*transactionrecord.BitmarkIssue); ok {
				allAssets[tx.AssetId] = struct{}{}
			}
		}
	}

	// save pending assets
backup_loop:
	for assetId := range allAssets {
		packedAsset := asset.Get(assetId)
		if nil == packedAsset {
			globalData.log.Errorf("asset [%s]: not in pending buffer", assetId)
			continue backup_loop // skip the corresponding issue since asset is corrupt
		}
		err := writeRecord(f, taggedTransaction, packedAsset)
		if nil != err {
			return err
		}
	}
	return nil
}

// pack up a proof record
func packProof(payId pay.PayId, nonce PayNonce) []byte {

	lp := len(payId)
	ln := len(nonce)
	packed := make([]byte, lp+ln)
	copy(packed[:], payId[:])
	copy(packed[lp:], nonce[:])

	return packed
}

// write a tagged block record
func writeBlock(f *os.File, tag tagType, txs []*transactionData) error {
	buffer := make([]byte, 0, 65535)
	for _, tx := range txs {
		buffer = append(buffer, tx.packed...)
	}
	return writeRecord(f, tag, buffer)
}

// write a tagged record
func writeRecord(f *os.File, tag tagType, packed []byte) error {

	if len(packed) > 65535 {
		globalData.log.Criticalf("write record packed length: %d > 65535", len(packed))
		logger.Panicf("write record packed length: %d > 65535", len(packed))
	}

	_, err := f.Write([]byte{byte(tag)})
	if nil != err {
		return err
	}

	count := make([]byte, 2)
	binary.BigEndian.PutUint16(count, uint16(len(packed)))
	_, err = f.Write(count)
	if nil != err {
		return err
	}
	_, err = f.Write(packed)
	return err
}

func readRecord(f *os.File) (tagType, transactionrecord.Packed, error) {

	tag := make([]byte, 1)
	n, err := f.Read(tag)
	if nil != err {
		return taggedEOF, []byte{}, err
	}
	if 1 != n {
		return taggedEOF, []byte{}, fmt.Errorf("read record name: read: %d, expected: %d", n, 1)
	}

	countBuffer := make([]byte, 2)
	n, err = f.Read(countBuffer)
	if nil != err {
		return taggedEOF, []byte{}, err
	}
	if 2 != n {
		return taggedEOF, []byte{}, fmt.Errorf("read record key count: read: %d, expected: %d", n, 2)
	}

	count := int(binary.BigEndian.Uint16(countBuffer))

	if count > 0 {
		buffer := make([]byte, count)
		n, err := f.Read(buffer)
		if nil != err {
			return taggedEOF, []byte{}, err
		}
		if count != n {
			return taggedEOF, []byte{}, fmt.Errorf("read record read: %d, expected: %d", n, count)
		}
		return tagType(tag[0]), buffer, nil
	}
	return tagType(tag[0]), []byte{}, nil
}
<commit_msg>[reservoir] fix spelling<commit_after>// SPDX-License-Identifier: ISC
// Copyright (c) 2014-2019 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package reservoir

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"os"

	"github.com/bitmark-inc/bitmarkd/asset"
	"github.com/bitmark-inc/bitmarkd/fault"
	"github.com/bitmark-inc/bitmarkd/mode"
	"github.com/bitmark-inc/bitmarkd/pay"
	"github.com/bitmark-inc/bitmarkd/storage"
	"github.com/bitmark-inc/bitmarkd/transactionrecord"
	"github.com/bitmark-inc/logger"
)

type tagType byte

// record types in cache file
const (
	taggedBOF         tagType = iota
	taggedEOF         tagType = iota
	taggedTransaction tagType = iota
	taggedProof       tagType = iota
)

// the BOF tag to check file version
// exact match is required
var bofData = []byte("bitmark-cache v1.0")

// Handles - storage handles used when restore from cache file
type Handles struct {
	Assets            storage.Handle
	BlockOwnerPayment storage.Handle
	Transaction       storage.Handle
	OwnerTx           storage.Handle
	OwnerData         storage.Handle
	Share             storage.Handle
	ShareQuantity     storage.Handle
}

// LoadFromFile - load transactions from file
// called later when system is able to handle the tx and proofs
func LoadFromFile(handles Handles) error {
	Disable()
	defer Enable()

	log := globalData.log

	log.Info("starting…")

	f, err := os.Open(globalData.filename)
	if nil != err {
		return err
	}
	defer f.Close()

	// must have BOF record first
	tag, packed, err := readRecord(f)
	if nil != err {
		return err
	}

	if taggedBOF != tag {
		return fmt.Errorf("expected BOF: %d but read: %d", taggedBOF, tag)
	}

	if !bytes.Equal(bofData, packed) {
		return fmt.Errorf("expected BOF: %q but read: %q", bofData, packed)
	}

	log.Infof("restore from file: %s", globalData.filename)

restore_loop:
	for {
		tag, packed, err := readRecord(f)
		if nil != err {
			log.Errorf("read record with error: %s\n", err)
			continue restore_loop
		}

		switch tag {

		case taggedEOF:
			break restore_loop

		case taggedTransaction:
			unpacked, _, err := packed.Unpack(mode.IsTesting())
			if nil != err {
				log.Errorf("unable to unpack asset: %s", err)
				continue restore_loop
			}

			restorer, err := NewTransactionRestorer(unpacked, packed, handles)
			if nil != err {
				log.Errorf("create transaction restorer with error: %s", err)
				continue restore_loop
			}

			err = restorer.Restore()
			if nil != err {
				log.Errorf("restore %s with error: %s", restorer, err)
				continue restore_loop
			}

		case taggedProof:
			var payId pay.PayId
			pn := len(payId)
			if len(packed) <= pn {
				log.Errorf("unable to unpack proof: record too short: %d  expected > %d", len(packed), pn)
				continue restore_loop
			}
			copy(payId[:], packed[:pn])
			nonce := packed[pn:]
			TryProof(payId, nonce)

		default:
			// in case any unsupported tag exist
			msg := fmt.Errorf("abort, read invalid tag: 0x%02x", tag)
			log.Error(msg.Error())
			return msg
		}
	}
	log.Info("restore completed")
	return nil
}

// save transactions to file
func saveToFile() error {
	globalData.Lock()
	defer globalData.Unlock()

	log := globalData.log

	if !globalData.initialised {
		log.Error("save when not initialised")
		return fault.NotInitialised
	}

	log.Info("saving…")

	f, err := os.OpenFile(globalData.filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
	if nil != err {
		return err
	}
	defer f.Close()

	// write beginning of file marker
	err = writeRecord(f, taggedBOF, bofData)
	if nil != err {
		return err
	}

	// all assets at start of file
	err = backupAssets(f)
	if nil != err {
		return err
	}

	// verified

	for _, item := range globalData.verifiedTransactions {
		err := writeRecord(f, taggedTransaction, item.packed)
		if nil != err {
			return err
		}
	}
	for _, item := range globalData.verifiedFreeIssues {
		err := writeBlock(f, taggedTransaction, item.txs)
		if nil != err {
			return err
		}
		err = writeRecord(f, taggedProof, packProof(item.payId, item.nonce))
		if nil != err {
			return err
		}
	}
	for _, item := range globalData.verifiedPaidIssues {
		err := writeBlock(f, taggedTransaction, item.txs)
		if nil != err {
			return err
		}
	}

	// pending

	for _, item := range globalData.pendingTransactions {
		err := writeRecord(f, taggedTransaction, item.tx.packed)
		if nil != err {
			return err
		}
	}
	for _, item := range globalData.pendingFreeIssues {
		err := writeBlock(f, taggedTransaction, item.txs)
		if nil != err {
			return err
		}
		err = writeRecord(f, taggedProof, packProof(item.payId, item.nonce))
		if nil != err {
			return err
		}
	}
	for _, item := range globalData.pendingPaidIssues {
		err := writeBlock(f, taggedTransaction, item.txs)
		if nil != err {
			return err
		}
	}

	// end the file
	err = writeRecord(f, taggedEOF, []byte("EOF"))
	if nil != err {
		return err
	}

	log.Info("save completed")
	return nil
}

func backupAssets(f *os.File) error {
	allAssets := make(map[transactionrecord.AssetIdentifier]struct{})

	// verified

	for _, item := range globalData.verifiedFreeIssues {
		for _, tx := range item.txs {
			if tx, ok := tx.transaction.(*transactionrecord.BitmarkIssue); ok {
				allAssets[tx.AssetId] = struct{}{}
			}
		}
	}
	for _, item := range globalData.verifiedPaidIssues {
		for _, tx := range item.txs {
			if tx, ok := tx.transaction.(*transactionrecord.BitmarkIssue); ok {
				allAssets[tx.AssetId] = struct{}{}
			}
		}
	}

	// pending

	for _, item := range globalData.pendingFreeIssues {
		for _, tx := range item.txs {
			if tx, ok := tx.transaction.(*transactionrecord.BitmarkIssue); ok {
				allAssets[tx.AssetId] = struct{}{}
			}
		}
	}
	for _, item := range globalData.pendingPaidIssues {
		for _, tx := range item.txs {
			if tx, ok := tx.transaction.(*transactionrecord.BitmarkIssue); ok {
				allAssets[tx.AssetId] = struct{}{}
			}
		}
	}

	// save pending assets
backup_loop:
	for assetId := range allAssets {
		packedAsset := asset.Get(assetId)
		if nil == packedAsset {
			globalData.log.Errorf("asset [%s]: not in pending buffer", assetId)
			continue backup_loop // skip the corresponding issue since asset is corrupt
		}
		err := writeRecord(f, taggedTransaction, packedAsset)
		if nil != err {
			return err
		}
	}
	return nil
}

// pack up a proof record
func packProof(payId pay.PayId, nonce PayNonce) []byte {

	lp := len(payId)
	ln := len(nonce)
	packed := make([]byte, lp+ln)
	copy(packed[:], payId[:])
	copy(packed[lp:], nonce[:])

	return packed
}

// write a tagged block record
func writeBlock(f *os.File, tag tagType, txs []*transactionData) error {
	buffer := make([]byte, 0, 65535)
	for _, tx := range txs {
		buffer = append(buffer, tx.packed...)
	}
	return writeRecord(f, tag, buffer)
}

// write a tagged record
func writeRecord(f *os.File, tag tagType, packed []byte) error {

	if len(packed) > 65535 {
		globalData.log.Criticalf("write record packed length: %d > 65535", len(packed))
		logger.Panicf("write record packed length: %d > 65535", len(packed))
	}

	_, err := f.Write([]byte{byte(tag)})
	if nil != err {
		return err
	}

	count := make([]byte, 2)
	binary.BigEndian.PutUint16(count, uint16(len(packed)))
	_, err = f.Write(count)
	if nil != err {
		return err
	}
	_, err = f.Write(packed)
	return err
}

func readRecord(f *os.File) (tagType, transactionrecord.Packed, error) {

	tag := make([]byte, 1)
	n, err := f.Read(tag)
	if nil != err {
		return taggedEOF, []byte{}, err
	}
	if 1 != n {
		return taggedEOF, []byte{}, fmt.Errorf("read record name: read: %d, expected: %d", n, 1)
	}

	countBuffer := make([]byte, 2)
	n, err = f.Read(countBuffer)
	if nil != err {
		return taggedEOF, []byte{}, err
	}
	if 2 != n {
		return taggedEOF, []byte{}, fmt.Errorf("read record key count: read: %d, expected: %d", n, 2)
	}

	count := int(binary.BigEndian.Uint16(countBuffer))

	if count > 0 {
		buffer := make([]byte, count)
		n, err := f.Read(buffer)
		if nil != err {
			return taggedEOF, []byte{}, err
		}
		if count != n {
			return taggedEOF, []byte{}, fmt.Errorf("read record read: %d, expected: %d", n, count)
		}
		return tagType(tag[0]), buffer, nil
	}
	return tagType(tag[0]), []byte{}, nil
}
<|endoftext|>


<commit_before>package core

import (
	"reflect"
	"strconv"
	"testing"

	"github.com/af83/edwig/model"
)

func createTestPartnerManager() *PartnerManager {
	referentials := NewMemoryReferentials()
	referential := referentials.New(ReferentialSlug("referential"))
	referentials.Save(referential)
	return NewPartnerManager(referential)
}

func Test_Partner_Id(t *testing.T) {
	partner := Partner{
		id: "6ba7b814-9dad-11d1-0-00c04fd430c8",
	}

	if expected := PartnerId("6ba7b814-9dad-11d1-0-00c04fd430c8"); partner.Id() != expected {
		t.Errorf("Partner.Id() returns wrong value, got: %s, required: %s", partner.Id(), expected)
	}
}

func Test_Partner_Slug(t *testing.T) {
	partner := Partner{
		slug: "partner",
	}

	if expected := PartnerSlug("partner"); partner.Slug() != expected {
		t.Errorf("Partner.Slug() returns wrong value, got: %s, required: %s", partner.Id(), expected)
	}
}

func Test_Partner_OperationnalStatus(t *testing.T) {
	partner := NewPartner()

	if expected := OPERATIONNAL_STATUS_UNKNOWN; partner.OperationnalStatus() != expected {
		t.Errorf("Partner.OperationnalStatus() returns wrong status, got: %s, required: %s", partner.OperationnalStatus(), expected)
	}
}

func Test_Partner_MarshalJSON(t *testing.T) {
	partner := Partner{
		id:                 "6ba7b814-9dad-11d1-0-00c04fd430c8",
		operationnalStatus: OPERATIONNAL_STATUS_UNKNOWN,
		slug:               "partner",
		Settings:           make(map[string]string),
		ConnectorTypes:     []string{},
	}
	expected := `{"ConnectorTypes":[],"Id":"6ba7b814-9dad-11d1-0-00c04fd430c8","OperationnalStatus":"unknown","Settings":{},"Slug":"partner"}`
	jsonBytes, err := partner.MarshalJSON()
	if err != nil {
		t.Fatal(err)
	}

	jsonString := string(jsonBytes)
	if jsonString != expected {
		t.Errorf("Partner.MarshalJSON() returns wrong json:\n got: %s\n want: %s", jsonString, expected)
	}
}

func Test_Partner_Save(t *testing.T) {
	partners := createTestPartnerManager()
	partner := partners.New("partner")

	if partner.manager != partners {
		t.Errorf("New partner manager should be partners")
	}

	ok := partner.Save()
	if !ok {
		t.Errorf("partner.Save() should succeed")
	}
	partner = partners.Find(partner.Id())
	if partner == nil {
		t.Errorf("New Partner should be found in Partners manager")
	}
}

func Test_Partner_RefreshConnectors(t *testing.T) {
	partner := Partner{connectors: make(map[string]Connector)}
	partner.RefreshConnectors()
	if partner.CheckStatusClient() != nil {
		t.Errorf("Partner CheckStatus client should be nil, got: %v", reflect.TypeOf(partner.CheckStatusClient()))
	}

	partner.ConnectorTypes = []string{"siri-check-status-client"}
	partner.RefreshConnectors()
	if _, ok := partner.CheckStatusClient().(*SIRICheckStatusClient); !ok {
		t.Errorf("Partner CheckStatus client should be SIRICheckStatusClient, got: %v", reflect.TypeOf(partner.CheckStatusClient()))
	}

	partner.ConnectorTypes = []string{"test-check-status-client"}
	partner.RefreshConnectors()
	if _, ok := partner.CheckStatusClient().(*TestCheckStatusClient); !ok {
		t.Errorf("Partner CheckStatus client should be TestCheckStatusClient, got: %v", reflect.TypeOf(partner.CheckStatusClient()))
	}
}

func Test_Partner_CanCollectTrue(t *testing.T) {
	partner := &Partner{}
	partner.Settings = make(map[string]string)
	stopAreaObjectId := model.NewObjectID("internal", "NINOXE:StopPoint:SP:24:LOC")

	partner.Settings["collect.include_stop_areas"] = "NINOXE:StopPoint:SP:24:LOC"
	if partner.CanCollect(stopAreaObjectId) != true {
		t.Errorf("Partner can collect should return true")
	}
}

func Test_Partner_CanCollectTrue2(t *testing.T) {
	partner := &Partner{}
	partner.Settings = make(map[string]string)
	stopAreaObjectId := model.NewObjectID("internal", "NINOXE:StopPoint:SP:24:LOC")

	if partner.CanCollect(stopAreaObjectId) != true {
		t.Errorf("Partner can collect should return true")
	}
}

func Test_Partner_CanCollectFalse(t *testing.T) {
	partner := &Partner{}
	partner.Settings = make(map[string]string)
	stopAreaObjectId := model.NewObjectID("internal", "BAD_VALUE")

	partner.Settings["collect.include_stop_areas"] = "NINOXE:StopPoint:SP:24:LOC"
	if partner.CanCollect(stopAreaObjectId) != false {
		t.Errorf("Partner can collect should return flase")
	}
}

func Test_Partners_FindAllByCollectPriority(t *testing.T) {
	partners := createTestPartnerManager()
	partner1 := Partner{}
	partner2 := Partner{}

	partner1.Settings = make(map[string]string)
	partner2.Settings = make(map[string]string)

	partner1.Settings["collect.priority"] = "2"
	partner1.SetSlug("First")

	partner2.Settings["collect.priority"] = "1"
	partner2.SetSlug("Second")

	partners.Save(&partner1)
	partners.Save(&partner2)

	orderedPartners := partners.FindAllByCollectPriority()
	if orderedPartners[0].Slug() != "First" {
		t.Errorf("Partners should be ordered")
	}
}

func Test_APIPartner_SetFactories(t *testing.T) {
	partner := &APIPartner{
		ConnectorTypes: []string{"unexistant-factory", "test-check-status-client"},
		factories:      make(map[string]ConnectorFactory),
	}
	partner.setFactories()

	if len(partner.factories) != 1 {
		t.Errorf("Factories should have been successfully created by setFactories")
	}
}

func Test_APIPartner_Validate(t *testing.T) {
	partners := createTestPartnerManager()
	// Check empty Slug
	apiPartner := &APIPartner{
		manager: partners,
	}
	valid := apiPartner.Validate()

	if valid {
		t.Errorf("Validate should return false")
	}
	if len(apiPartner.Errors) != 1 {
		t.Errorf("apiPartner Errors should not be empty")
	}
	if len(apiPartner.Errors["Slug"]) != 1 || apiPartner.Errors["Slug"][0] != ERROR_BLANK {
		t.Errorf("apiPartner should have Error for Slug, got %v", apiPartner.Errors)
	}

	// Check Already Used Slug and local_credential
	partner := partners.New("slug")
	partner.Settings["local_credential"] = "cred"
	partners.Save(partner)
	apiPartner = &APIPartner{
		Slug:     "slug",
		Settings: map[string]string{"local_credential": "cred"},
		manager:  partners,
	}
	valid = apiPartner.Validate()

	if valid {
		t.Errorf("Validate should return false")
	}
	if len(apiPartner.Errors) != 2 {
		t.Errorf("apiPartner Errors should not be empty")
	}
	if len(apiPartner.Errors["Slug"]) != 1 || apiPartner.Errors["Slug"][0] != ERROR_UNIQUE {
		t.Errorf("apiPartner should have Error for Slug, got %v", apiPartner.Errors)
	}
	if len(apiPartner.Errors["Settings[\"local_credential\"]"]) != 1 || apiPartner.Errors["Settings[\"local_credential\"]"][0] != ERROR_UNIQUE {
		t.Errorf("apiPartner should have Error for local_credential, got %v", apiPartner.Errors)
	}

	// Check ok
	apiPartner = &APIPartner{
		Slug:     "slug2",
		Settings: map[string]string{"local_credential": "cred2"},
		manager:  partners,
	}
	valid = apiPartner.Validate()

	if !valid {
		t.Errorf("Validate should return true")
	}
	if len(apiPartner.Errors) != 0 {
		t.Errorf("apiPartner Errors should be empty")
	}
}

func Test_NewPartnerManager(t *testing.T) {
	partners := createTestPartnerManager()

	if partners.guardian == nil {
		t.Errorf("New PartnerManager should have a PartnersGuardian")
	}
}

func Test_PartnerManager_New(t *testing.T) {
	partners := createTestPartnerManager()
	partner := partners.New("partner")

	if partner.Id() != "" {
		t.Errorf("New Partner identifier should be an empty string, got: %s", partner.Id())
	}
}

func Test_PartnerManager_Save(t *testing.T) {
	partners := createTestPartnerManager()
	partner := partners.New("partner")

	if success := partners.Save(partner); !success {
		t.Errorf("Save should return true")
	}

	if partner.Id() == "" {
		t.Errorf("New Partner identifier should not be an empty string")
	}
}

func Test_PartnerManager_Find_NotFound(t *testing.T) {
	partners := createTestPartnerManager()
	partner := partners.Find("6ba7b814-9dad-11d1-0-00c04fd430c8")
	if partner != nil {
		t.Errorf("Find should return false when Partner isn't found")
	}
}

func Test_PartnerManager_Find(t *testing.T) {
	partners := createTestPartnerManager()

	existingPartner := partners.New("partner")
	partners.Save(existingPartner)
	partnerId := existingPartner.Id()

	partner := partners.Find(partnerId)
	if partner == nil {
		t.Fatal("Find should return true when Partner is found")
	}
	if partner.Id() != partnerId {
		t.Errorf("Find should return a Partner with the given Id")
	}
}

func Test_PartnerManager_FindByCredential(t *testing.T) {
	partners := createTestPartnerManager()

	existingPartner := partners.New("partner")
	existingPartner.Settings["local_credential"] = "cred"
	partners.Save(existingPartner)

	partner, ok := partners.FindByLocalCredential("cred")
	if !ok {
		t.Fatal("FindByLocalCredential should return true when Partner is found")
	}
	if partner.Id() != existingPartner.Id() {
		t.Errorf("FindByLocalCredential should return a Partner with the given local_credential")
	}
}

func Test_PartnerManager_FindAll(t *testing.T) {
	partners := createTestPartnerManager()

	for i := 0; i < 5; i++ {
		existingPartner := partners.New(PartnerSlug(strconv.Itoa(i)))
		partners.Save(existingPartner)
	}

	foundPartners := partners.FindAll()

	if len(foundPartners) != 5 {
		t.Errorf("FindAll should return all partners")
	}
}

func Test_PartnerManager_Delete(t *testing.T) {
	partners := createTestPartnerManager()

	existingPartner := partners.New("partner")
	partners.Save(existingPartner)

	partnerId := existingPartner.Id()

	partners.Delete(existingPartner)

	partner := partners.Find(partnerId)
	if partner != nil {
		t.Errorf("Deleted Partner should not be findable")
	}
}

func Test_MemoryPartners_Load(t *testing.T) {
	model.InitTestDb(t)
	defer model.CleanTestDb(t)

	referentials := NewMemoryReferentials()
	referential := referentials.New("referential")
	referentials.Save(referential)

	// Insert Data in the test db
	var databasePartner = struct {
		Id             string `db:"id"`
		ReferentialId  string `db:"referential_id"`
		Slug           string `db:"slug"`
		Settings       string `db:"settings"`
		ConnectorTypes string `db:"connector_types"`
	}{
		Id:             "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11",
		ReferentialId:  string(referential.Id()),
		Slug:           "ratp",
		Settings:       "{}",
		ConnectorTypes: "[]",
	}

	model.Database.AddTableWithName(databasePartner, "partners")
	err := model.Database.Insert(&databasePartner)
	if err != nil {
		t.Fatal(err)
	}

	// Fetch data from the db
	partners := NewPartnerManager(referential)
	err = partners.Load()
	if err != nil {
		t.Fatal(err)
	}

	partnerId := PartnerId(databasePartner.Id)
	partner := partners.Find(partnerId)
	if partner == nil {
		t.Errorf("Loaded Partners should be found")
	} else if partner.Id() != partnerId {
		t.Errorf("Wrong Id:\n got: %v\n expected: %v", partner.Id(), partnerId)
	}
}
<commit_msg>Refs #2936 Ajout de tests<commit_after>package core

import (
	"reflect"
	"strconv"
	"testing"

	"github.com/af83/edwig/model"
)

func createTestPartnerManager() *PartnerManager {
	referentials := NewMemoryReferentials()
	referential := referentials.New(ReferentialSlug("referential"))
	referentials.Save(referential)
	return NewPartnerManager(referential)
}

func Test_Partner_Id(t *testing.T) {
	partner := Partner{
		id: "6ba7b814-9dad-11d1-0-00c04fd430c8",
	}

	if expected := PartnerId("6ba7b814-9dad-11d1-0-00c04fd430c8"); partner.Id() != expected {
		t.Errorf("Partner.Id() returns wrong value, got: %s, required: %s", partner.Id(), expected)
	}
}

func Test_Partner_Slug(t *testing.T) {
	partner := Partner{
		slug: "partner",
	}

	if expected := PartnerSlug("partner"); partner.Slug() != expected {
		t.Errorf("Partner.Slug() returns wrong value, got: %s, required: %s", partner.Id(), expected)
	}
}

func Test_Partner_OperationnalStatus(t *testing.T) {
	partner := NewPartner()

	if expected := OPERATIONNAL_STATUS_UNKNOWN; partner.OperationnalStatus() != expected {
		t.Errorf("Partner.OperationnalStatus() returns wrong status, got: %s, required: %s", partner.OperationnalStatus(), expected)
	}
}

func Test_Partner_MarshalJSON(t *testing.T) {
	partner := Partner{
		id:                 "6ba7b814-9dad-11d1-0-00c04fd430c8",
		operationnalStatus: OPERATIONNAL_STATUS_UNKNOWN,
		slug:               "partner",
		Settings:           make(map[string]string),
		ConnectorTypes:     []string{},
	}
	expected := `{"ConnectorTypes":[],"Id":"6ba7b814-9dad-11d1-0-00c04fd430c8","OperationnalStatus":"unknown","Settings":{},"Slug":"partner"}`
	jsonBytes, err := partner.MarshalJSON()
	if err != nil {
		t.Fatal(err)
	}

	jsonString := string(jsonBytes)
	if jsonString != expected {
		t.Errorf("Partner.MarshalJSON() returns wrong json:\n got: %s\n want: %s", jsonString, expected)
	}
}

func Test_Partner_Save(t *testing.T) {
	partners := createTestPartnerManager()
	partner := partners.New("partner")

	if partner.manager != partners {
		t.Errorf("New partner manager should be partners")
	}

	ok := partner.Save()
	if !ok {
		t.Errorf("partner.Save() should succeed")
	}
	partner = partners.Find(partner.Id())
	if partner == nil {
		t.Errorf("New Partner should be found in Partners manager")
	}
}

func Test_Partner_RefreshConnectors(t *testing.T) {
	partner := Partner{connectors: make(map[string]Connector)}
	partner.RefreshConnectors()
	if partner.CheckStatusClient() != nil {
		t.Errorf("Partner CheckStatus client should be nil, got: %v", reflect.TypeOf(partner.CheckStatusClient()))
	}

	partner.ConnectorTypes = []string{"siri-check-status-client"}
	partner.RefreshConnectors()
	if _, ok := partner.CheckStatusClient().(*SIRICheckStatusClient); !ok {
		t.Errorf("Partner CheckStatus client should be SIRICheckStatusClient, got: %v", reflect.TypeOf(partner.CheckStatusClient()))
	}

	partner.ConnectorTypes = []string{"test-check-status-client"}
	partner.RefreshConnectors()
	if _, ok := partner.CheckStatusClient().(*TestCheckStatusClient); !ok {
		t.Errorf("Partner CheckStatus client should be TestCheckStatusClient, got: %v", reflect.TypeOf(partner.CheckStatusClient()))
	}
}

func Test_Partner_CanCollectTrue(t *testing.T) {
	partner := &Partner{}
	partner.Settings = make(map[string]string)
	stopAreaObjectId := model.NewObjectID("internal", "NINOXE:StopPoint:SP:24:LOC")

	partner.Settings["collect.include_stop_areas"] = "NINOXE:StopPoint:SP:24:LOC"
	if partner.CanCollect(stopAreaObjectId) != true {
		t.Errorf("Partner can collect should return true")
	}
}

func Test_Partner_RemoteObjectIDKindPresent(t *testing.T) {
	partner := &Partner{}
	partner.Settings = make(map[string]string)

	partner.Settings["siri-stop-monitoring-request-broadcaster.remote_objectid_kind"] = "Kind1"
	partner.Settings["remote_objectid_kind"] = "Kind2"

	if partner.RemoteObjectIDKind() != "Kind1" {
		t.Errorf("RemoteObjectIDKind should be egals to Kind1")
	}
}

func Test_Partner_RemoteObjectIDKindAbsent(t *testing.T) {
	partner := &Partner{}
	partner.Settings = make(map[string]string)

	partner.Settings["siri-stop-monitoring-request-broadcaster.remote_objectid_kind"] = ""
	partner.Settings["remote_objectid_kind"] = "Kind2"

	if partner.RemoteObjectIDKind() != "Kind2" {
		t.Errorf("RemoteObjectIDKind should be egals to Kind2")
	}
}

func Test_Partner_CanCollectTrue2(t *testing.T) {
	partner := &Partner{}
	partner.Settings = make(map[string]string)
	stopAreaObjectId := model.NewObjectID("internal", "NINOXE:StopPoint:SP:24:LOC")

	if partner.CanCollect(stopAreaObjectId) != true {
		t.Errorf("Partner can collect should return true")
	}
}

func Test_Partner_CanCollectFalse(t *testing.T) {
	partner := &Partner{}
	partner.Settings = make(map[string]string)
	stopAreaObjectId := model.NewObjectID("internal", "BAD_VALUE")

	partner.Settings["collect.include_stop_areas"] = "NINOXE:StopPoint:SP:24:LOC"
	if partner.CanCollect(stopAreaObjectId) != false {
		t.Errorf("Partner can collect should return flase")
	}
}

func Test_Partners_FindAllByCollectPriority(t *testing.T) {
	partners := createTestPartnerManager()
	partner1 := Partner{}
	partner2 := Partner{}

	partner1.Settings = make(map[string]string)
	partner2.Settings = make(map[string]string)

	partner1.Settings["collect.priority"] = "2"
	partner1.SetSlug("First")

	partner2.Settings["collect.priority"] = "1"
	partner2.SetSlug("Second")

	partners.Save(&partner1)
	partners.Save(&partner2)

	orderedPartners := partners.FindAllByCollectPriority()
	if orderedPartners[0].Slug() != "First" {
		t.Errorf("Partners should be ordered")
	}
}

func Test_APIPartner_SetFactories(t *testing.T) {
	partner := &APIPartner{
		ConnectorTypes: []string{"unexistant-factory", "test-check-status-client"},
		factories:      make(map[string]ConnectorFactory),
	}
	partner.setFactories()

	if len(partner.factories) != 1 {
		t.Errorf("Factories should have been successfully created by setFactories")
	}
}

func Test_APIPartner_Validate(t *testing.T) {
	partners := createTestPartnerManager()
	// Check empty Slug
	apiPartner := &APIPartner{
		manager: partners,
	}
	valid := apiPartner.Validate()

	if valid {
		t.Errorf("Validate should return false")
	}
	if len(apiPartner.Errors) != 1 {
		t.Errorf("apiPartner Errors should not be empty")
	}
	if len(apiPartner.Errors["Slug"]) != 1 || apiPartner.Errors["Slug"][0] != ERROR_BLANK {
		t.Errorf("apiPartner should have Error for Slug, got %v", apiPartner.Errors)
	}

	// Check Already Used Slug and local_credential
	partner := partners.New("slug")
	partner.Settings["local_credential"] = "cred"
	partners.Save(partner)
	apiPartner = &APIPartner{
		Slug:     "slug",
		Settings: map[string]string{"local_credential": "cred"},
		manager:  partners,
	}
	valid = apiPartner.Validate()

	if valid {
		t.Errorf("Validate should return false")
	}
	if len(apiPartner.Errors) != 2 {
		t.Errorf("apiPartner Errors should not be empty")
	}
	if len(apiPartner.Errors["Slug"]) != 1 || apiPartner.Errors["Slug"][0] != ERROR_UNIQUE {
		t.Errorf("apiPartner should have Error for Slug, got %v", apiPartner.Errors)
	}
	if len(apiPartner.Errors["Settings[\"local_credential\"]"]) != 1 || apiPartner.Errors["Settings[\"local_credential\"]"][0] != ERROR_UNIQUE {
		t.Errorf("apiPartner should have Error for local_credential, got %v", apiPartner.Errors)
	}

	// Check ok
	apiPartner = &APIPartner{
		Slug:     "slug2",
		Settings: map[string]string{"local_credential": "cred2"},
		manager:  partners,
	}
	valid = apiPartner.Validate()

	if !valid {
		t.Errorf("Validate should return true")
	}
	if len(apiPartner.Errors) != 0 {
		t.Errorf("apiPartner Errors should be empty")
	}
}

func Test_NewPartnerManager(t *testing.T) {
	partners := createTestPartnerManager()

	if partners.guardian == nil {
		t.Errorf("New PartnerManager should have a PartnersGuardian")
	}
}

func Test_PartnerManager_New(t *testing.T) {
	partners := createTestPartnerManager()
	partner := partners.New("partner")

	if partner.Id() != "" {
		t.Errorf("New Partner identifier should be an empty string, got: %s", partner.Id())
	}
}

func Test_PartnerManager_Save(t *testing.T) {
	partners := createTestPartnerManager()
	partner := partners.New("partner")

	if success := partners.Save(partner); !success {
		t.Errorf("Save should return true")
	}

	if partner.Id() == "" {
		t.Errorf("New Partner identifier should not be an empty string")
	}
}

func Test_PartnerManager_Find_NotFound(t *testing.T) {
	partners := createTestPartnerManager()
	partner := partners.Find("6ba7b814-9dad-11d1-0-00c04fd430c8")
	if partner != nil {
		t.Errorf("Find should return false when Partner isn't found")
	}
}

func Test_PartnerManager_Find(t *testing.T) {
	partners := createTestPartnerManager()

	existingPartner := partners.New("partner")
	partners.Save(existingPartner)
	partnerId := existingPartner.Id()

	partner := partners.Find(partnerId)
	if partner == nil {
		t.Fatal("Find should return true when Partner is found")
	}
	if partner.Id() != partnerId {
		t.Errorf("Find should return a Partner with the given Id")
	}
}

func Test_PartnerManager_FindByCredential(t *testing.T) {
	partners := createTestPartnerManager()

	existingPartner := partners.New("partner")
	existingPartner.Settings["local_credential"] = "cred"
	partners.Save(existingPartner)

	partner, ok := partners.FindByLocalCredential("cred")
	if !ok {
		t.Fatal("FindByLocalCredential should return true when Partner is found")
	}
	if partner.Id() != existingPartner.Id() {
		t.Errorf("FindByLocalCredential should return a Partner with the given local_credential")
	}
}

func Test_PartnerManager_FindAll(t *testing.T) {
	partners := createTestPartnerManager()

	for i := 0; i < 5; i++ {
		existingPartner := partners.New(PartnerSlug(strconv.Itoa(i)))
		partners.Save(existingPartner)
	}

	foundPartners := partners.FindAll()

	if len(foundPartners) != 5 {
		t.Errorf("FindAll should return all partners")
	}
}

func Test_PartnerManager_Delete(t *testing.T) {
	partners := createTestPartnerManager()

	existingPartner := partners.New("partner")
	partners.Save(existingPartner)

	partnerId := existingPartner.Id()

	partners.Delete(existingPartner)

	partner := partners.Find(partnerId)
	if partner != nil {
		t.Errorf("Deleted Partner should not be findable")
	}
}

func Test_MemoryPartners_Load(t *testing.T) {
	model.InitTestDb(t)
	defer model.CleanTestDb(t)

	referentials := NewMemoryReferentials()
	referential := referentials.New("referential")
	referentials.Save(referential)

	// Insert Data in the test db
	var databasePartner = struct {
		Id             string `db:"id"`
		ReferentialId  string `db:"referential_id"`
		Slug           string `db:"slug"`
		Settings       string `db:"settings"`
		ConnectorTypes string `db:"connector_types"`
	}{
		Id:             "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11",
		ReferentialId:  string(referential.Id()),
		Slug:           "ratp",
		Settings:       "{}",
		ConnectorTypes: "[]",
	}

	model.Database.AddTableWithName(databasePartner, "partners")
	err := model.Database.Insert(&databasePartner)
	if err != nil {
		t.Fatal(err)
	}

	// Fetch data from the db
	partners := NewPartnerManager(referential)
	err = partners.Load()
	if err != nil {
		t.Fatal(err)
	}

	partnerId := PartnerId(databasePartner.Id)
	partner := partners.Find(partnerId)
	if partner == nil {
		t.Errorf("Loaded Partners should be found")
	} else if partner.Id() != partnerId {
		t.Errorf("Wrong Id:\n got: %v\n expected: %v", partner.Id(), partnerId)
	}
}
<|endoftext|>


<commit_before>package root

import (
	"encoding/json"
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudformation"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/kubernetes-incubator/kube-aws/cfnstack"
	controlplane "github.com/kubernetes-incubator/kube-aws/core/controlplane/cluster"
	controlplane_cfg "github.com/kubernetes-incubator/kube-aws/core/controlplane/config"
	nodepool "github.com/kubernetes-incubator/kube-aws/core/nodepool/cluster"
	nodepool_cfg "github.com/kubernetes-incubator/kube-aws/core/nodepool/config"
	"github.com/kubernetes-incubator/kube-aws/core/root/config"
	"github.com/kubernetes-incubator/kube-aws/core/root/defaults"
	"github.com/kubernetes-incubator/kube-aws/filereader/jsontemplate"
	model "github.com/kubernetes-incubator/kube-aws/model"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"
)

const (
	LOCAL_ROOT_STACK_TEMPLATE_PATH = defaults.RootStackTemplateTmplFile
	REMOTE_STACK_TEMPLATE_FILENAME = "stack.json"
)

func (c clusterImpl) Export() error {
	assets, err := c.Assets()

	if err != nil {
		return err
	}

	for _, asset := range assets.AsMap() {
		path := filepath.Join("exported", "stacks", asset.Path)
		fmt.Printf("Exporting %s\n", path)
		dir := filepath.Dir(path)
		if err := os.MkdirAll(dir, 0700); err != nil {
			return fmt.Errorf("failed to create directory \"%s\": %v", dir, err)
		}
		if err := ioutil.WriteFile(path, []byte(asset.Content), 0600); err != nil {
			return fmt.Errorf("Error writing %s : %v", path, err)
		}
		if strings.HasSuffix(path, "stack.json") && c.controlPlane.KMSKeyARN == "" {
			fmt.Printf("BEWARE: %s contains your TLS secrets!\n", path)
		}
	}
	return nil
}

func (c clusterImpl) EstimateCost() ([]string, error) {

	cfSvc := cloudformation.New(c.session)
	var urls []string

	controlPlaneTemplate, err := c.controlPlane.RenderStackTemplateAsString()

	if err != nil {
		return nil, fmt.Errorf("failed to render control plane template %v", err)
	}

	controlPlaneCost, err := c.stackProvisioner().EstimateTemplateCost(cfSvc, controlPlaneTemplate, nil)

	if err != nil {
		return nil, fmt.Errorf("failed to estimate cost for control plane %v", err)
	}

	urls = append(urls, *controlPlaneCost.Url)

	for i, p := range c.nodePools {
		nodePoolsTemplate, err := p.RenderStackTemplateAsString()

		if err != nil {
			return nil, fmt.Errorf("failed to render node pool #%d template: %v", i, err)
		}

		nodePoolsCost, err := c.stackProvisioner().EstimateTemplateCost(cfSvc, nodePoolsTemplate, []*cloudformation.Parameter{
			{
				ParameterKey:   aws.String("ControlPlaneStackName"),
				ParameterValue: aws.String("fake-name"),
			},
		})

		if err != nil {
			return nil, fmt.Errorf("failed to estimate cost for node pool #%d %v", i, err)
		}

		urls = append(urls, *nodePoolsCost.Url)
	}

	return urls, nil

}

type Cluster interface {
	Assets() (cfnstack.Assets, error)
	Create() error
	Export() error
	EstimateCost() ([]string, error)
	Info() (*Info, error)
	Update() (string, error)
	ValidateStack() (string, error)
	ValidateTemplates() error
	ControlPlane() *controlplane.Cluster
}

func ClusterFromFile(configPath string, opts options, awsDebug bool) (Cluster, error) {
	cfg, err := config.ConfigFromFile(configPath)
	if err != nil {
		return nil, err
	}
	return ClusterFromConfig(cfg, opts, awsDebug)
}

func ClusterFromConfig(cfg *config.Config, opts options, awsDebug bool) (Cluster, error) {
	cpOpts := controlplane_cfg.StackTemplateOptions{
		AssetsDir:             opts.AssetsDir,
		ControllerTmplFile:    opts.ControllerTmplFile,
		EtcdTmplFile:          opts.EtcdTmplFile,
		StackTemplateTmplFile: opts.ControlPlaneStackTemplateTmplFile,
		PrettyPrint:           opts.PrettyPrint,
		S3URI:                 opts.S3URI,
		SkipWait:              opts.SkipWait,
	}
	cp, err := controlplane.NewCluster(cfg.Cluster, cpOpts, awsDebug)
	if err != nil {
		return nil, err
	}
	nodePools := []*nodepool.Cluster{}
	for i, c := range cfg.NodePools {
		npOpts := nodepool_cfg.StackTemplateOptions{
			AssetsDir:             opts.AssetsDir,
			WorkerTmplFile:        opts.WorkerTmplFile,
			StackTemplateTmplFile: opts.NodePoolStackTemplateTmplFile,
			PrettyPrint:           opts.PrettyPrint,
			S3URI:                 opts.S3URI,
			SkipWait:              opts.SkipWait,
		}
		np, err := nodepool.NewCluster(c, npOpts, awsDebug)
		if err != nil {
			return nil, fmt.Errorf("failed to load node pool #%d: %v", i, err)
		}
		nodePools = append(nodePools, np)
	}
	awsConfig := aws.NewConfig().
		WithRegion(cfg.Region.String()).
		WithCredentialsChainVerboseErrors(true)

	if awsDebug {
		awsConfig = awsConfig.WithLogLevel(aws.LogDebug)
	}

	session, err := session.NewSession(awsConfig)
	if err != nil {
		return nil, fmt.Errorf("failed to establish aws session: %v", err)
	}
	return clusterImpl{
		opts:         opts,
		controlPlane: cp,
		nodePools:    nodePools,
		session:      session,
	}, nil
}

type clusterImpl struct {
	controlPlane *controlplane.Cluster
	nodePools    []*nodepool.Cluster
	opts         options
	session      *session.Session
}

func (c clusterImpl) ControlPlane() *controlplane.Cluster {
	return c.controlPlane
}

func (c clusterImpl) Create() error {
	cfSvc := cloudformation.New(c.session)

	stackTemplateURL, err := c.prepareTemplateWithAssets()
	if err != nil {
		return err
	}

	if c.controlPlane.CloudWatchLogging.Enabled && c.controlPlane.CloudWatchLogging.LocalStreaming.Enabled {
		// Return Journald logs in a separate GoRoutine
		quit := make(chan bool)
		defer func() { quit <- true }()
		go streamJournaldLogs(c, quit)
	}

	return c.stackProvisioner().CreateStackAtURLAndWait(cfSvc, stackTemplateURL)
}

func (c clusterImpl) Info() (*Info, error) {
	// TODO Cleaner way to obtain this dependency
	cpConfig, err := c.controlPlane.Cluster.Config()
	if err != nil {
		return nil, err
	}

	describer := NewClusterDescriber(c.controlPlane.ClusterName, c.stackName(), cpConfig, c.session)
	return describer.Info()
}

func (c clusterImpl) prepareTemplateWithAssets() (string, error) {
	assets, err := c.Assets()

	if err != nil {
		return "", err
	}

	s3Svc := s3.New(c.session)
	err = c.stackProvisioner().UploadAssets(s3Svc, assets)
	if err != nil {
		return "", err
	}

	asset, err := assets.FindAssetByStackAndFileName(c.stackName(), REMOTE_STACK_TEMPLATE_FILENAME)

	if err != nil {
		return "", fmt.Errorf("failed to prepare template with assets: %v", err)
	}

	return asset.URL()
}

func (c clusterImpl) Assets() (cfnstack.Assets, error) {
	stackTemplate, err := c.renderTemplateAsString()
	if err != nil {
		return nil, fmt.Errorf("Error while rendering template : %v", err)
	}
	s3URI := fmt.Sprintf("%s/kube-aws/clusters/%s/exported/stacks",
		strings.TrimSuffix(c.opts.S3URI, "/"),
		c.controlPlane.ClusterName,
	)

	assetsBuilder := cfnstack.NewAssetsBuilder(c.stackName(), s3URI, c.controlPlane.Region)
	assetsBuilder.Add(REMOTE_STACK_TEMPLATE_FILENAME, stackTemplate)
	assets := assetsBuilder.Build()

	cpAssets := c.controlPlane.Assets()
	assets = assets.Merge(cpAssets)

	for _, np := range c.nodePools {
		a := np.Assets()
		assets = assets.Merge(a)
	}

	return assets, nil
}

func (c clusterImpl) templatePath() string {
	return c.opts.RootStackTemplateTmplFile
}

func (c clusterImpl) templateParams() TemplateParams {
	params := newTemplateParams(c)
	return params
}

func (c clusterImpl) renderTemplateAsString() (string, error) {
	template, err := jsontemplate.GetString(c.templatePath(), c.templateParams(), c.opts.PrettyPrint)
	if err != nil {
		return "", err
	}
	return template, nil
}

func (c clusterImpl) stackProvisioner() *cfnstack.Provisioner {
	stackPolicyBody := `{
  "Statement" : [
    {
       "Effect" : "Allow",
       "Principal" : "*",
       "Action" : "Update:*",
       "Resource" : "*"
     }
  ]
}`
	return cfnstack.NewProvisioner(
		c.stackName(),
		c.tags(),
		c.opts.S3URI,
		c.controlPlane.Region,
		stackPolicyBody,
		c.session)
}

func (c clusterImpl) stackName() string {
	return c.controlPlane.Cluster.ClusterName
}

func (c clusterImpl) tags() map[string]string {
	return c.controlPlane.Cluster.StackTags
}

func (c clusterImpl) Update() (string, error) {
	cfSvc := cloudformation.New(c.session)

	templateUrl, err := c.prepareTemplateWithAssets()
	if err != nil {
		return "", err
	}

	if c.controlPlane.CloudWatchLogging.Enabled && c.controlPlane.CloudWatchLogging.LocalStreaming.Enabled {
		// Return Journald logs in a separate GoRoutine
		quit := make(chan bool)
		defer func() { quit <- true }()
		go streamJournaldLogs(c, quit)
	}

	return c.stackProvisioner().UpdateStackAtURLAndWait(cfSvc, templateUrl)
}

func (c clusterImpl) ValidateTemplates() error {
	_, err := c.renderTemplateAsString()
	if err != nil {
		return fmt.Errorf("failed to validate template: %v", err)
	}
	if _, err := c.controlPlane.RenderStackTemplateAsString(); err != nil {
		return fmt.Errorf("failed to validate control plane template: %v", err)
	}
	for i, p := range c.nodePools {
		if _, err := p.RenderStackTemplateAsString(); err != nil {
			return fmt.Errorf("failed to validate node pool #%d template: %v", i, err)
		}
	}
	return nil
}

// ValidateStack validates all the CloudFormation stack templates already uploaded to S3
func (c clusterImpl) ValidateStack() (string, error) {
	reports := []string{}

	// Upload all the assets including stack templates and cloud-configs for all the stacks
	rootStackTemplateURL, err := c.prepareTemplateWithAssets()
	if err != nil {
		return "", err
	}

	r, err := c.stackProvisioner().ValidateStackAtURL(rootStackTemplateURL)
	if err != nil {
		return "", err
	}

	reports = append(reports, r)

	cpReport, err := c.controlPlane.ValidateStack()
	if err != nil {
		return "", fmt.Errorf("failed to validate control plane: %v", err)
	}
	reports = append(reports, cpReport)

	for i, p := range c.nodePools {
		npReport, err := p.ValidateStack()
		if err != nil {
			return "", fmt.Errorf("failed to validate node pool #%d: %v", i, err)
		}
		reports = append(reports, npReport)
	}

	return strings.Join(reports, "\n"), nil
}

func streamJournaldLogs(c clusterImpl, quit chan bool) error {
	fmt.Printf("Printing filtered Journald logs for log group '%s'...\nNOTE: Due to high initial entropy, failures may occur during the early stages of booting.\n", c.controlPlane.ClusterName)
	cwlSvc := cloudwatchlogs.New(c.session)
	startTime := time.Now().Unix() * 1E3
	fleInput := cloudwatchlogs.FilterLogEventsInput{
		LogGroupName:  &c.controlPlane.ClusterName,
		FilterPattern: &c.controlPlane.CloudWatchLogging.LocalStreaming.Filter,
		StartTime:     &startTime}
	messages := make(map[string]int64)

	for {
		select {
		case <-quit:
			return nil
		default:
			out, err := cwlSvc.FilterLogEvents(&fleInput)
			if err != nil {
				continue
			}
			if len(out.Events) > 1 {
				startTime = *out.Events[len(out.Events)-1].Timestamp
				for _, event := range out.Events {
					if *event.Timestamp > messages[*event.Message]+c.controlPlane.CloudWatchLogging.LocalStreaming.Interval() {
						messages[*event.Message] = *event.Timestamp
						res := model.SystemdMessageResponse{}
						json.Unmarshal([]byte(*event.Message), &res)
						fmt.Printf("%s: \"%s\"\n", res.Hostname, res.Message)
					}
				}
			}
			fleInput = cloudwatchlogs.FilterLogEventsInput{
				LogGroupName:  &c.controlPlane.ClusterName,
				FilterPattern: &c.controlPlane.CloudWatchLogging.LocalStreaming.Filter,
				NextToken:     out.NextToken,
				StartTime:     &startTime}
		}
	}
}
<commit_msg>Add 1 second sleep between AWS API calls<commit_after>package root

import (
	"encoding/json"
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudformation"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/kubernetes-incubator/kube-aws/cfnstack"
	controlplane "github.com/kubernetes-incubator/kube-aws/core/controlplane/cluster"
	controlplane_cfg "github.com/kubernetes-incubator/kube-aws/core/controlplane/config"
	nodepool "github.com/kubernetes-incubator/kube-aws/core/nodepool/cluster"
	nodepool_cfg "github.com/kubernetes-incubator/kube-aws/core/nodepool/config"
	"github.com/kubernetes-incubator/kube-aws/core/root/config"
	"github.com/kubernetes-incubator/kube-aws/core/root/defaults"
	"github.com/kubernetes-incubator/kube-aws/filereader/jsontemplate"
	model "github.com/kubernetes-incubator/kube-aws/model"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"
)

const (
	LOCAL_ROOT_STACK_TEMPLATE_PATH = defaults.RootStackTemplateTmplFile
	REMOTE_STACK_TEMPLATE_FILENAME = "stack.json"
)

func (c clusterImpl) Export() error {
	assets, err := c.Assets()

	if err != nil {
		return err
	}

	for _, asset := range assets.AsMap() {
		path := filepath.Join("exported", "stacks", asset.Path)
		fmt.Printf("Exporting %s\n", path)
		dir := filepath.Dir(path)
		if err := os.MkdirAll(dir, 0700); err != nil {
			return fmt.Errorf("failed to create directory \"%s\": %v", dir, err)
		}
		if err := ioutil.WriteFile(path, []byte(asset.Content), 0600); err != nil {
			return fmt.Errorf("Error writing %s : %v", path, err)
		}
		if strings.HasSuffix(path, "stack.json") && c.controlPlane.KMSKeyARN == "" {
			fmt.Printf("BEWARE: %s contains your TLS secrets!\n", path)
		}
	}
	return nil
}

func (c clusterImpl) EstimateCost() ([]string, error) {

	cfSvc := cloudformation.New(c.session)
	var urls []string

	controlPlaneTemplate, err := c.controlPlane.RenderStackTemplateAsString()

	if err != nil {
		return nil, fmt.Errorf("failed to render control plane template %v", err)
	}

	controlPlaneCost, err := c.stackProvisioner().EstimateTemplateCost(cfSvc, controlPlaneTemplate, nil)

	if err != nil {
		return nil, fmt.Errorf("failed to estimate cost for control plane %v", err)
	}

	urls = append(urls, *controlPlaneCost.Url)

	for i, p := range c.nodePools {
		nodePoolsTemplate, err := p.RenderStackTemplateAsString()

		if err != nil {
			return nil, fmt.Errorf("failed to render node pool #%d template: %v", i, err)
		}

		nodePoolsCost, err := c.stackProvisioner().EstimateTemplateCost(cfSvc, nodePoolsTemplate, []*cloudformation.Parameter{
			{
				ParameterKey:   aws.String("ControlPlaneStackName"),
				ParameterValue: aws.String("fake-name"),
			},
		})

		if err != nil {
			return nil, fmt.Errorf("failed to estimate cost for node pool #%d %v", i, err)
		}

		urls = append(urls, *nodePoolsCost.Url)
	}

	return urls, nil

}

type Cluster interface {
	Assets() (cfnstack.Assets, error)
	Create() error
	Export() error
	EstimateCost() ([]string, error)
	Info() (*Info, error)
	Update() (string, error)
	ValidateStack() (string, error)
	ValidateTemplates() error
	ControlPlane() *controlplane.Cluster
}

func ClusterFromFile(configPath string, opts options, awsDebug bool) (Cluster, error) {
	cfg, err := config.ConfigFromFile(configPath)
	if err != nil {
		return nil, err
	}
	return ClusterFromConfig(cfg, opts, awsDebug)
}

func ClusterFromConfig(cfg *config.Config, opts options, awsDebug bool) (Cluster, error) {
	cpOpts := controlplane_cfg.StackTemplateOptions{
		AssetsDir:             opts.AssetsDir,
		ControllerTmplFile:    opts.ControllerTmplFile,
		EtcdTmplFile:          opts.EtcdTmplFile,
		StackTemplateTmplFile: opts.ControlPlaneStackTemplateTmplFile,
		PrettyPrint:           opts.PrettyPrint,
		S3URI:                 opts.S3URI,
		SkipWait:              opts.SkipWait,
	}
	cp, err := controlplane.NewCluster(cfg.Cluster, cpOpts, awsDebug)
	if err != nil {
		return nil, err
	}
	nodePools := []*nodepool.Cluster{}
	for i, c := range cfg.NodePools {
		npOpts := nodepool_cfg.StackTemplateOptions{
			AssetsDir:             opts.AssetsDir,
			WorkerTmplFile:        opts.WorkerTmplFile,
			StackTemplateTmplFile: opts.NodePoolStackTemplateTmplFile,
			PrettyPrint:           opts.PrettyPrint,
			S3URI:                 opts.S3URI,
			SkipWait:              opts.SkipWait,
		}
		np, err := nodepool.NewCluster(c, npOpts, awsDebug)
		if err != nil {
			return nil, fmt.Errorf("failed to load node pool #%d: %v", i, err)
		}
		nodePools = append(nodePools, np)
	}
	awsConfig := aws.NewConfig().
		WithRegion(cfg.Region.String()).
		WithCredentialsChainVerboseErrors(true)

	if awsDebug {
		awsConfig = awsConfig.WithLogLevel(aws.LogDebug)
	}

	session, err := session.NewSession(awsConfig)
	if err != nil {
		return nil, fmt.Errorf("failed to establish aws session: %v", err)
	}
	return clusterImpl{
		opts:         opts,
		controlPlane: cp,
		nodePools:    nodePools,
		session:      session,
	}, nil
}

type clusterImpl struct {
	controlPlane *controlplane.Cluster
	nodePools    []*nodepool.Cluster
	opts         options
	session      *session.Session
}

func (c clusterImpl) ControlPlane() *controlplane.Cluster {
	return c.controlPlane
}

func (c clusterImpl) Create() error {
	cfSvc := cloudformation.New(c.session)

	stackTemplateURL, err := c.prepareTemplateWithAssets()
	if err != nil {
		return err
	}

	if c.controlPlane.CloudWatchLogging.Enabled && c.controlPlane.CloudWatchLogging.LocalStreaming.Enabled {
		// Return Journald logs in a separate GoRoutine
		quit := make(chan bool)
		defer func() { quit <- true }()
		go streamJournaldLogs(c, quit)
	}

	return c.stackProvisioner().CreateStackAtURLAndWait(cfSvc, stackTemplateURL)
}

func (c clusterImpl) Info() (*Info, error) {
	// TODO Cleaner way to obtain this dependency
	cpConfig, err := c.controlPlane.Cluster.Config()
	if err != nil {
		return nil, err
	}

	describer := NewClusterDescriber(c.controlPlane.ClusterName, c.stackName(), cpConfig, c.session)
	return describer.Info()
}

func (c clusterImpl) prepareTemplateWithAssets() (string, error) {
	assets, err := c.Assets()

	if err != nil {
		return "", err
	}

	s3Svc := s3.New(c.session)
	err = c.stackProvisioner().UploadAssets(s3Svc, assets)
	if err != nil {
		return "", err
	}

	asset, err := assets.FindAssetByStackAndFileName(c.stackName(), REMOTE_STACK_TEMPLATE_FILENAME)

	if err != nil {
		return "", fmt.Errorf("failed to prepare template with assets: %v", err)
	}

	return asset.URL()
}

func (c clusterImpl) Assets() (cfnstack.Assets, error) {
	stackTemplate, err := c.renderTemplateAsString()
	if err != nil {
		return nil, fmt.Errorf("Error while rendering template : %v", err)
	}
	s3URI := fmt.Sprintf("%s/kube-aws/clusters/%s/exported/stacks",
		strings.TrimSuffix(c.opts.S3URI, "/"),
		c.controlPlane.ClusterName,
	)

	assetsBuilder := cfnstack.NewAssetsBuilder(c.stackName(), s3URI, c.controlPlane.Region)
	assetsBuilder.Add(REMOTE_STACK_TEMPLATE_FILENAME, stackTemplate)
	assets := assetsBuilder.Build()

	cpAssets := c.controlPlane.Assets()
	assets = assets.Merge(cpAssets)

	for _, np := range c.nodePools {
		a := np.Assets()
		assets = assets.Merge(a)
	}

	return assets, nil
}

func (c clusterImpl) templatePath() string {
	return c.opts.RootStackTemplateTmplFile
}

func (c clusterImpl) templateParams() TemplateParams {
	params := newTemplateParams(c)
	return params
}

func (c clusterImpl) renderTemplateAsString() (string, error) {
	template, err := jsontemplate.GetString(c.templatePath(), c.templateParams(), c.opts.PrettyPrint)
	if err != nil {
		return "", err
	}
	return template, nil
}

func (c clusterImpl) stackProvisioner() *cfnstack.Provisioner {
	stackPolicyBody := `{
  "Statement" : [
    {
       "Effect" : "Allow",
       "Principal" : "*",
       "Action" : "Update:*",
       "Resource" : "*"
     }
  ]
}`
	return cfnstack.NewProvisioner(
		c.stackName(),
		c.tags(),
		c.opts.S3URI,
		c.controlPlane.Region,
		stackPolicyBody,
		c.session)
}

func (c clusterImpl) stackName() string {
	return c.controlPlane.Cluster.ClusterName
}

func (c clusterImpl) tags() map[string]string {
	return c.controlPlane.Cluster.StackTags
}

func (c clusterImpl) Update() (string, error) {
	cfSvc := cloudformation.New(c.session)

	templateUrl, err := c.prepareTemplateWithAssets()
	if err != nil {
		return "", err
	}

	if c.controlPlane.CloudWatchLogging.Enabled && c.controlPlane.CloudWatchLogging.LocalStreaming.Enabled {
		// Return Journald logs in a separate GoRoutine
		quit := make(chan bool)
		defer func() { quit <- true }()
		go streamJournaldLogs(c, quit)
	}

	return c.stackProvisioner().UpdateStackAtURLAndWait(cfSvc, templateUrl)
}

func (c clusterImpl) ValidateTemplates() error {
	_, err := c.renderTemplateAsString()
	if err != nil {
		return fmt.Errorf("failed to validate template: %v", err)
	}
	if _, err := c.controlPlane.RenderStackTemplateAsString(); err != nil {
		return fmt.Errorf("failed to validate control plane template: %v", err)
	}
	for i, p := range c.nodePools {
		if _, err := p.RenderStackTemplateAsString(); err != nil {
			return fmt.Errorf("failed to validate node pool #%d template: %v", i, err)
		}
	}
	return nil
}

// ValidateStack validates all the CloudFormation stack templates already uploaded to S3
func (c clusterImpl) ValidateStack() (string, error) {
	reports := []string{}

	// Upload all the assets including stack templates and cloud-configs for all the stacks
	rootStackTemplateURL, err := c.prepareTemplateWithAssets()
	if err != nil {
		return "", err
	}

	r, err := c.stackProvisioner().ValidateStackAtURL(rootStackTemplateURL)
	if err != nil {
		return "", err
	}

	reports = append(reports, r)

	cpReport, err := c.controlPlane.ValidateStack()
	if err != nil {
		return "", fmt.Errorf("failed to validate control plane: %v", err)
	}
	reports = append(reports, cpReport)

	for i, p := range c.nodePools {
		npReport, err := p.ValidateStack()
		if err != nil {
			return "", fmt.Errorf("failed to validate node pool #%d: %v", i, err)
		}
		reports = append(reports, npReport)
	}

	return strings.Join(reports, "\n"), nil
}

func streamJournaldLogs(c clusterImpl, quit chan bool) error {
	fmt.Printf("Printing filtered Journald logs for log group '%s'...\nNOTE: Due to high initial entropy, failures may occur during the early stages of booting.\n", c.controlPlane.ClusterName)
	cwlSvc := cloudwatchlogs.New(c.session)
	startTime := time.Now().Unix() * 1E3
	fleInput := cloudwatchlogs.FilterLogEventsInput{
		LogGroupName:  &c.controlPlane.ClusterName,
		FilterPattern: &c.controlPlane.CloudWatchLogging.LocalStreaming.Filter,
		StartTime:     &startTime}
	messages := make(map[string]int64)

	for {
		select {
		case <-quit:
			return nil
		default:
			out, err := cwlSvc.FilterLogEvents(&fleInput)
			if err != nil {
				continue
			}
			if len(out.Events) > 1 {
				startTime = *out.Events[len(out.Events)-1].Timestamp
				for _, event := range out.Events {
					if *event.Timestamp > messages[*event.Message]+c.controlPlane.CloudWatchLogging.LocalStreaming.Interval() {
						messages[*event.Message] = *event.Timestamp
						res := model.SystemdMessageResponse{}
						json.Unmarshal([]byte(*event.Message), &res)
						fmt.Printf("%s: \"%s\"\n", res.Hostname, res.Message)
					}
				}
			}
			fleInput = cloudwatchlogs.FilterLogEventsInput{
				LogGroupName:  &c.controlPlane.ClusterName,
				FilterPattern: &c.controlPlane.CloudWatchLogging.LocalStreaming.Filter,
				NextToken:     out.NextToken,
				StartTime:     &startTime}
		}
		time.Sleep(time.Second)
	}
}
<|endoftext|>


<commit_before>package contractor

import (
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

// Constants related to fees and fee estimation.
const (
	// estimatedFileContractTransactionSize provides the estimated size of
	// the average file contract in bytes.
	estimatedFileContractTransactionSize = 1200
)

// Constants related to contract formation parameters.
var (
	// To alleviate potential block propagation issues, the contractor sleeps
	// between each contract formation.
	contractFormationInterval = build.Select(build.Var{
		Dev:      10 * time.Second,
		Standard: 60 * time.Second,
		Testing:  10 * time.Millisecond,
	}).(time.Duration)

	// minContractFundRenewalThreshold defines the ratio of remaining funds to
	// total contract cost below which the contractor will prematurely renew a
	// contract.
	minContractFundRenewalThreshold = float64(0.03) // 3%

	// minHostsForEstimations describes the minimum number of hosts that
	// are needed to make broad estimations such as the number of sectors
	// that you can store on the network for a given allowance.
	minHostsForEstimations = build.Select(build.Var{
		// The number is set lower than standard so that it can
		// be reached/exceeded easily within development
		// environments, but set high enough that it's also
		// easy to fall short within the development
		// environments.
		Dev: 5,
		// Hosts can have a lot of variance. Selecting too many
		// hosts will high-ball the price estimation, but users
		// shouldn't be selecting rewer hosts, and if there are
		// too few hosts being selected for estimation there is
		// a risk of underestimating the actual price, which is
		// something we'd rather avoid.
		Standard: 10,
		// Testing tries to happen as fast as possible,
		// therefore tends to run with a lot fewer hosts.
		Testing: 4,
	}).(int)

	// minScoreHostBuffer defines how many extra hosts are queried when trying
	// to figure out an appropriate minimum score for the hosts that we have.
	minScoreHostBuffer = build.Select(build.Var{
		Dev:      2,
		Standard: 10,
		Testing:  1,
	}).(int)
)

// Constants related to the safety values for when the contractor is forming
// contracts.
var (
	maxCollateral    = types.SiacoinPrecision.Mul64(1e3) // 1k SC
	maxDownloadPrice = maxStoragePrice.Mul64(3 * 4320)
	maxStoragePrice  = types.SiacoinPrecision.Mul64(30e3).Div(modules.BlockBytesPerMonthTerabyte) // 30k SC / TB / Month
	maxUploadPrice   = maxStoragePrice.Mul64(3 * 4320)                                            // 3 months of storage

	// scoreLeeway defines the factor by which a host can miss the goal score
	// for a set of hosts. To determine the goal score, a new set of hosts is
	// queried from the hostdb and the lowest scoring among them is selected.
	// That score is then divided by scoreLeeway to get the minimum score that a
	// host is allowed to have before being marked as !GoodForUpload.
	scoreLeeway = types.NewCurrency64(25)
)
<commit_msg>increase scoreLeeway<commit_after>package contractor

import (
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

// Constants related to fees and fee estimation.
const (
	// estimatedFileContractTransactionSize provides the estimated size of
	// the average file contract in bytes.
	estimatedFileContractTransactionSize = 1200
)

// Constants related to contract formation parameters.
var (
	// To alleviate potential block propagation issues, the contractor sleeps
	// between each contract formation.
	contractFormationInterval = build.Select(build.Var{
		Dev:      10 * time.Second,
		Standard: 60 * time.Second,
		Testing:  10 * time.Millisecond,
	}).(time.Duration)

	// minContractFundRenewalThreshold defines the ratio of remaining funds to
	// total contract cost below which the contractor will prematurely renew a
	// contract.
	minContractFundRenewalThreshold = float64(0.03) // 3%

	// minHostsForEstimations describes the minimum number of hosts that
	// are needed to make broad estimations such as the number of sectors
	// that you can store on the network for a given allowance.
	minHostsForEstimations = build.Select(build.Var{
		// The number is set lower than standard so that it can
		// be reached/exceeded easily within development
		// environments, but set high enough that it's also
		// easy to fall short within the development
		// environments.
		Dev: 5,
		// Hosts can have a lot of variance. Selecting too many
		// hosts will high-ball the price estimation, but users
		// shouldn't be selecting rewer hosts, and if there are
		// too few hosts being selected for estimation there is
		// a risk of underestimating the actual price, which is
		// something we'd rather avoid.
		Standard: 10,
		// Testing tries to happen as fast as possible,
		// therefore tends to run with a lot fewer hosts.
		Testing: 4,
	}).(int)

	// minScoreHostBuffer defines how many extra hosts are queried when trying
	// to figure out an appropriate minimum score for the hosts that we have.
	minScoreHostBuffer = build.Select(build.Var{
		Dev:      2,
		Standard: 10,
		Testing:  1,
	}).(int)
)

// Constants related to the safety values for when the contractor is forming
// contracts.
var (
	maxCollateral    = types.SiacoinPrecision.Mul64(1e3) // 1k SC
	maxDownloadPrice = maxStoragePrice.Mul64(3 * 4320)
	maxStoragePrice  = types.SiacoinPrecision.Mul64(30e3).Div(modules.BlockBytesPerMonthTerabyte) // 30k SC / TB / Month
	maxUploadPrice   = maxStoragePrice.Mul64(3 * 4320)                                            // 3 months of storage

	// scoreLeeway defines the factor by which a host can miss the goal score
	// for a set of hosts. To determine the goal score, a new set of hosts is
	// queried from the hostdb and the lowest scoring among them is selected.
	// That score is then divided by scoreLeeway to get the minimum score that a
	// host is allowed to have before being marked as !GoodForUpload.
	scoreLeeway = types.NewCurrency64(100)
)
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awsutil"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/schema"
)

// AWS Route resource Schema declaration
func resourceAwsRoute() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsRouteCreate,
		Read:   resourceAwsRouteRead,
		Update: resourceAwsRouteUpdate,
		Delete: resourceAwsRouteDelete,
		Exists: resourceAwsRouteExists,

		Schema: map[string]*schema.Schema{
			"destination_cidr_block": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},

			"destination_prefix_list_id": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"gateway_id": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},

			"instance_id": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},

			"instance_owner_id": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"network_interface_id": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},

			"origin": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"state": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"route_table_id": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},

			"vpc_peering_connection_id": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},
		},
	}
}

func resourceAwsRouteCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn
	var numTargets int
	var setTarget string
	allowedTargets := []string{
		"gateway_id",
		"instance_id",
		"network_interface_id",
		"vpc_peering_connection_id",
	}

	// Check if more than 1 target is specified
	for _, target := range allowedTargets {
		if len(d.Get(target).(string)) > 0 {
			numTargets++
			setTarget = target
		}
	}

	if numTargets > 1 {
		fmt.Errorf("Error: more than 1 target specified. Only 1 of gateway_id" +
			"instance_id, network_interface_id, route_table_id or" +
			"vpc_peering_connection_id is allowed.")
	}

	createOpts := &ec2.CreateRouteInput{}
	// Formulate CreateRouteInput based on the target type
	switch setTarget {
	case "gateway_id":
		createOpts = &ec2.CreateRouteInput{
			RouteTableId:         aws.String(d.Get("route_table_id").(string)),
			DestinationCidrBlock: aws.String(d.Get("destination_cidr_block").(string)),
			GatewayId:            aws.String(d.Get("gateway_id").(string)),
		}
	case "instance_id":
		createOpts = &ec2.CreateRouteInput{
			RouteTableId:         aws.String(d.Get("route_table_id").(string)),
			DestinationCidrBlock: aws.String(d.Get("destination_cidr_block").(string)),
			InstanceId:           aws.String(d.Get("instance_id").(string)),
		}
	case "network_interface_id":
		createOpts = &ec2.CreateRouteInput{
			RouteTableId:         aws.String(d.Get("route_table_id").(string)),
			DestinationCidrBlock: aws.String(d.Get("destination_cidr_block").(string)),
			NetworkInterfaceId:   aws.String(d.Get("network_interface_id").(string)),
		}
	case "vpc_peering_connection_id":
		createOpts = &ec2.CreateRouteInput{
			RouteTableId:           aws.String(d.Get("route_table_id").(string)),
			DestinationCidrBlock:   aws.String(d.Get("destination_cidr_block").(string)),
			VpcPeeringConnectionId: aws.String(d.Get("vpc_peering_connection_id").(string)),
		}
	default:
		fmt.Errorf("Error: invalid target type specified.")
	}
	log.Printf("[DEBUG] Route create config: %s", awsutil.Prettify(createOpts))

	// Create the route
	_, err := conn.CreateRoute(createOpts)
	if err != nil {
		return fmt.Errorf("Error creating route: %s", err)
	}

	route, err := findResourceRoute(conn, d.Get("route_table_id").(string), d.Get("destination_cidr_block").(string))
	if err != nil {
		fmt.Errorf("Error: %s", awsutil.Prettify(err))
	}

	d.SetId(routeIDHash(d, route))

	return resourceAwsRouteRead(d, meta)
}

func resourceAwsRouteRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn
	route, err := findResourceRoute(conn, d.Get("route_table_id").(string), d.Get("destination_cidr_block").(string))
	if err != nil {
		return err
	}

	d.Set("destination_prefix_list_id", route.DestinationPrefixListId)
	d.Set("gateway_id", route.GatewayId)
	d.Set("instance_id", route.InstanceId)
	d.Set("instance_owner_id", route.InstanceOwnerId)
	d.Set("network_interface_id", route.NetworkInterfaceId)
	d.Set("origin", route.Origin)
	d.Set("state", route.State)
	d.Set("vpc_peering_connection_id", route.VpcPeeringConnectionId)

	return nil
}

func resourceAwsRouteUpdate(d *schema.ResourceData, meta interface{}) error {
	if d.HasChange("destination_cidr_block") {
		return resourceAwsRouteRecreate(d, meta)
	}

	conn := meta.(*AWSClient).ec2conn
	var numTargets int
	var setTarget string
	allowedTargets := []string{
		"gateway_id",
		"instance_id",
		"network_interface_id",
		"vpc_peering_connection_id",
	}
	replaceOpts := &ec2.ReplaceRouteInput{}

	// Check if more than 1 target is specified
	for _, target := range allowedTargets {
		if len(d.Get(target).(string)) > 0 {
			numTargets++
			setTarget = target
		}
	}

	if numTargets > 1 {
		fmt.Errorf("Error: more than 1 target specified. Only 1 of gateway_id" +
			"instance_id, network_interface_id, route_table_id or" +
			"vpc_peering_connection_id is allowed.")
	}

	// Formulate ReplaceRouteInput based on the target type
	switch setTarget {
	case "gateway_id":
		replaceOpts = &ec2.ReplaceRouteInput{
			RouteTableId:         aws.String(d.Get("route_table_id").(string)),
			DestinationCidrBlock: aws.String(d.Get("destination_cidr_block").(string)),
			GatewayId:            aws.String(d.Get("gateway_id").(string)),
		}
	case "instance_id":
		replaceOpts = &ec2.ReplaceRouteInput{
			RouteTableId:         aws.String(d.Get("route_table_id").(string)),
			DestinationCidrBlock: aws.String(d.Get("destination_cidr_block").(string)),
			InstanceId:           aws.String(d.Get("instance_id").(string)),
			//NOOP: Ensure we don't blow away network interface id that is set after instance is launched
			NetworkInterfaceId:	  aws.String(d.Get("network_interface_id").(string)),
		}
	case "network_interface_id":
		replaceOpts = &ec2.ReplaceRouteInput{
			RouteTableId:         aws.String(d.Get("route_table_id").(string)),
			DestinationCidrBlock: aws.String(d.Get("destination_cidr_block").(string)),
			NetworkInterfaceId:   aws.String(d.Get("network_interface_id").(string)),
		}
	case "vpc_peering_connection_id":
		replaceOpts = &ec2.ReplaceRouteInput{
			RouteTableId:           aws.String(d.Get("route_table_id").(string)),
			DestinationCidrBlock:   aws.String(d.Get("destination_cidr_block").(string)),
			VpcPeeringConnectionId: aws.String(d.Get("vpc_peering_connection_id").(string)),
		}
	default:
		fmt.Errorf("Error: invalid target type specified.")
	}
	log.Printf("[DEBUG] Route replace config: %s", awsutil.Prettify(replaceOpts))

	// Replace the route
	_, err := conn.ReplaceRoute(replaceOpts)
	if err != nil {
		return err
	}

	return nil
}

func resourceAwsRouteRecreate(d *schema.ResourceData, meta interface{}) error {
	//Destination Cidr is used for identification
	// if changed, we should delete the old route, recreate the new route
	conn := meta.(*AWSClient).ec2conn

	oc, _ := d.GetChange("destination_cidr_block")

	var oldRtId interface{}
	if d.HasChange("route_table_id") {
		oldRtId, _ = d.GetChange("route_table_id")
	} else {
		oldRtId = d.Get("route_table_id")
	}

	if err := deleteAwsRoute(conn, oldRtId.(string), oc.(string)); err != nil {
		return err
	}
	d.SetId("")

	return resourceAwsRouteCreate(d, meta)
}

func resourceAwsRouteDelete(d *schema.ResourceData, meta interface{}) error {
	err := deleteAwsRoute(meta.(*AWSClient).ec2conn,
		d.Get("route_table_id").(string), d.Get("destination_cidr_block").(string))
	if err != nil {
		return err
	}

	d.SetId("")
	return nil
}

func resourceAwsRouteExists(d *schema.ResourceData, meta interface{}) (bool, error) {
	conn := meta.(*AWSClient).ec2conn
	routeTableId := d.Get("route_table_id").(string)

	findOpts := &ec2.DescribeRouteTablesInput{
		RouteTableIds: []*string{&routeTableId},
	}

	res, err := conn.DescribeRouteTables(findOpts)
	if err != nil {
		return false, err
	}

	cidr := d.Get("destination_cidr_block").(string)
	for _, route := range (*res.RouteTables[0]).Routes {
		if *route.DestinationCidrBlock == cidr {
			return true, nil
		}
	}

	return false, nil
}

// Create an ID for a route
func routeIDHash(d *schema.ResourceData, r *ec2.Route) string {
	return fmt.Sprintf("r-%s%d", d.Get("route_table_id").(string), hashcode.String(*r.DestinationCidrBlock))
}

// Helper: retrieve a route
func findResourceRoute(conn *ec2.EC2, rtbid string, cidr string) (*ec2.Route, error) {
	routeTableID := rtbid

	findOpts := &ec2.DescribeRouteTablesInput{
		RouteTableIds: []*string{&routeTableID},
	}

	resp, err := conn.DescribeRouteTables(findOpts)
	if err != nil {
		return nil, err
	}

	for _, route := range (*resp.RouteTables[0]).Routes {
		if *route.DestinationCidrBlock == cidr {
			return route, nil
		}
	}

	return nil, nil
}

func deleteAwsRoute(conn *ec2.EC2, routeTableId string, cidr string) error {
	deleteOpts := &ec2.DeleteRouteInput{
		RouteTableId:         aws.String(routeTableId),
		DestinationCidrBlock: aws.String(cidr),
	}
	log.Printf("[DEBUG] Route delete opts: %s", awsutil.Prettify(deleteOpts))

	resp, err := conn.DeleteRoute(deleteOpts)
	log.Printf("[DEBUG] Route delete result: %s", awsutil.Prettify(resp))
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>Removing usage of awsutil.Prettify.<commit_after>package aws

import (
	"fmt"
	"log"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/schema"
)

// AWS Route resource Schema declaration
func resourceAwsRoute() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsRouteCreate,
		Read:   resourceAwsRouteRead,
		Update: resourceAwsRouteUpdate,
		Delete: resourceAwsRouteDelete,
		Exists: resourceAwsRouteExists,

		Schema: map[string]*schema.Schema{
			"destination_cidr_block": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},

			"destination_prefix_list_id": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"gateway_id": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},

			"instance_id": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},

			"instance_owner_id": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"network_interface_id": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},

			"origin": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"state": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"route_table_id": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},

			"vpc_peering_connection_id": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},
		},
	}
}

func resourceAwsRouteCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn
	var numTargets int
	var setTarget string
	allowedTargets := []string{
		"gateway_id",
		"instance_id",
		"network_interface_id",
		"vpc_peering_connection_id",
	}

	// Check if more than 1 target is specified
	for _, target := range allowedTargets {
		if len(d.Get(target).(string)) > 0 {
			numTargets++
			setTarget = target
		}
	}

	if numTargets > 1 {
		fmt.Errorf("Error: more than 1 target specified. Only 1 of gateway_id" +
			"instance_id, network_interface_id, route_table_id or" +
			"vpc_peering_connection_id is allowed.")
	}

	createOpts := &ec2.CreateRouteInput{}
	// Formulate CreateRouteInput based on the target type
	switch setTarget {
	case "gateway_id":
		createOpts = &ec2.CreateRouteInput{
			RouteTableId:         aws.String(d.Get("route_table_id").(string)),
			DestinationCidrBlock: aws.String(d.Get("destination_cidr_block").(string)),
			GatewayId:            aws.String(d.Get("gateway_id").(string)),
		}
	case "instance_id":
		createOpts = &ec2.CreateRouteInput{
			RouteTableId:         aws.String(d.Get("route_table_id").(string)),
			DestinationCidrBlock: aws.String(d.Get("destination_cidr_block").(string)),
			InstanceId:           aws.String(d.Get("instance_id").(string)),
		}
	case "network_interface_id":
		createOpts = &ec2.CreateRouteInput{
			RouteTableId:         aws.String(d.Get("route_table_id").(string)),
			DestinationCidrBlock: aws.String(d.Get("destination_cidr_block").(string)),
			NetworkInterfaceId:   aws.String(d.Get("network_interface_id").(string)),
		}
	case "vpc_peering_connection_id":
		createOpts = &ec2.CreateRouteInput{
			RouteTableId:           aws.String(d.Get("route_table_id").(string)),
			DestinationCidrBlock:   aws.String(d.Get("destination_cidr_block").(string)),
			VpcPeeringConnectionId: aws.String(d.Get("vpc_peering_connection_id").(string)),
		}
	default:
		fmt.Errorf("Error: invalid target type specified.")
	}
	log.Printf("[DEBUG] Route create config: %s", createOpts)

	// Create the route
	_, err := conn.CreateRoute(createOpts)
	if err != nil {
		return fmt.Errorf("Error creating route: %s", err)
	}

	route, err := findResourceRoute(conn, d.Get("route_table_id").(string), d.Get("destination_cidr_block").(string))
	if err != nil {
		fmt.Errorf("Error: %s", err)
	}

	d.SetId(routeIDHash(d, route))

	return resourceAwsRouteRead(d, meta)
}

func resourceAwsRouteRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn
	route, err := findResourceRoute(conn, d.Get("route_table_id").(string), d.Get("destination_cidr_block").(string))
	if err != nil {
		return err
	}

	d.Set("destination_prefix_list_id", route.DestinationPrefixListId)
	d.Set("gateway_id", route.GatewayId)
	d.Set("instance_id", route.InstanceId)
	d.Set("instance_owner_id", route.InstanceOwnerId)
	d.Set("network_interface_id", route.NetworkInterfaceId)
	d.Set("origin", route.Origin)
	d.Set("state", route.State)
	d.Set("vpc_peering_connection_id", route.VpcPeeringConnectionId)

	return nil
}

func resourceAwsRouteUpdate(d *schema.ResourceData, meta interface{}) error {
	if d.HasChange("destination_cidr_block") {
		return resourceAwsRouteRecreate(d, meta)
	}

	conn := meta.(*AWSClient).ec2conn
	var numTargets int
	var setTarget string
	allowedTargets := []string{
		"gateway_id",
		"instance_id",
		"network_interface_id",
		"vpc_peering_connection_id",
	}
	replaceOpts := &ec2.ReplaceRouteInput{}

	// Check if more than 1 target is specified
	for _, target := range allowedTargets {
		if len(d.Get(target).(string)) > 0 {
			numTargets++
			setTarget = target
		}
	}

	if numTargets > 1 {
		fmt.Errorf("Error: more than 1 target specified. Only 1 of gateway_id" +
			"instance_id, network_interface_id, route_table_id or" +
			"vpc_peering_connection_id is allowed.")
	}

	// Formulate ReplaceRouteInput based on the target type
	switch setTarget {
	case "gateway_id":
		replaceOpts = &ec2.ReplaceRouteInput{
			RouteTableId:         aws.String(d.Get("route_table_id").(string)),
			DestinationCidrBlock: aws.String(d.Get("destination_cidr_block").(string)),
			GatewayId:            aws.String(d.Get("gateway_id").(string)),
		}
	case "instance_id":
		replaceOpts = &ec2.ReplaceRouteInput{
			RouteTableId:         aws.String(d.Get("route_table_id").(string)),
			DestinationCidrBlock: aws.String(d.Get("destination_cidr_block").(string)),
			InstanceId:           aws.String(d.Get("instance_id").(string)),
			//NOOP: Ensure we don't blow away network interface id that is set after instance is launched
			NetworkInterfaceId:	  aws.String(d.Get("network_interface_id").(string)),
		}
	case "network_interface_id":
		replaceOpts = &ec2.ReplaceRouteInput{
			RouteTableId:         aws.String(d.Get("route_table_id").(string)),
			DestinationCidrBlock: aws.String(d.Get("destination_cidr_block").(string)),
			NetworkInterfaceId:   aws.String(d.Get("network_interface_id").(string)),
		}
	case "vpc_peering_connection_id":
		replaceOpts = &ec2.ReplaceRouteInput{
			RouteTableId:           aws.String(d.Get("route_table_id").(string)),
			DestinationCidrBlock:   aws.String(d.Get("destination_cidr_block").(string)),
			VpcPeeringConnectionId: aws.String(d.Get("vpc_peering_connection_id").(string)),
		}
	default:
		fmt.Errorf("Error: invalid target type specified.")
	}
	log.Printf("[DEBUG] Route replace config: %s", replaceOpts)

	// Replace the route
	_, err := conn.ReplaceRoute(replaceOpts)
	if err != nil {
		return err
	}

	return nil
}

func resourceAwsRouteRecreate(d *schema.ResourceData, meta interface{}) error {
	//Destination Cidr is used for identification
	// if changed, we should delete the old route, recreate the new route
	conn := meta.(*AWSClient).ec2conn

	oc, _ := d.GetChange("destination_cidr_block")

	var oldRtId interface{}
	if d.HasChange("route_table_id") {
		oldRtId, _ = d.GetChange("route_table_id")
	} else {
		oldRtId = d.Get("route_table_id")
	}

	if err := deleteAwsRoute(conn, oldRtId.(string), oc.(string)); err != nil {
		return err
	}
	d.SetId("")

	return resourceAwsRouteCreate(d, meta)
}

func resourceAwsRouteDelete(d *schema.ResourceData, meta interface{}) error {
	err := deleteAwsRoute(meta.(*AWSClient).ec2conn,
		d.Get("route_table_id").(string), d.Get("destination_cidr_block").(string))
	if err != nil {
		return err
	}

	d.SetId("")
	return nil
}

func resourceAwsRouteExists(d *schema.ResourceData, meta interface{}) (bool, error) {
	conn := meta.(*AWSClient).ec2conn
	routeTableId := d.Get("route_table_id").(string)

	findOpts := &ec2.DescribeRouteTablesInput{
		RouteTableIds: []*string{&routeTableId},
	}

	res, err := conn.DescribeRouteTables(findOpts)
	if err != nil {
		return false, err
	}

	cidr := d.Get("destination_cidr_block").(string)
	for _, route := range (*res.RouteTables[0]).Routes {
		if *route.DestinationCidrBlock == cidr {
			return true, nil
		}
	}

	return false, nil
}

// Create an ID for a route
func routeIDHash(d *schema.ResourceData, r *ec2.Route) string {
	return fmt.Sprintf("r-%s%d", d.Get("route_table_id").(string), hashcode.String(*r.DestinationCidrBlock))
}

// Helper: retrieve a route
func findResourceRoute(conn *ec2.EC2, rtbid string, cidr string) (*ec2.Route, error) {
	routeTableID := rtbid

	findOpts := &ec2.DescribeRouteTablesInput{
		RouteTableIds: []*string{&routeTableID},
	}

	resp, err := conn.DescribeRouteTables(findOpts)
	if err != nil {
		return nil, err
	}

	for _, route := range (*resp.RouteTables[0]).Routes {
		if *route.DestinationCidrBlock == cidr {
			return route, nil
		}
	}

	return nil, nil
}

func deleteAwsRoute(conn *ec2.EC2, routeTableId string, cidr string) error {
	deleteOpts := &ec2.DeleteRouteInput{
		RouteTableId:         aws.String(routeTableId),
		DestinationCidrBlock: aws.String(cidr),
	}
	log.Printf("[DEBUG] Route delete opts: %s", deleteOpts)

	resp, err := conn.DeleteRoute(deleteOpts)
	log.Printf("[DEBUG] Route delete result: %s", resp)
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>