<commit_before>/*
Copyright 2020 The Tekton Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tektonpipeline

import (
	"context"
	"fmt"
	"os"
	"path/filepath"

	"github.com/go-logr/zapr"
	mfc "github.com/manifestival/client-go-client"
	mf "github.com/manifestival/manifestival"
	"github.com/tektoncd/operator/pkg/apis/operator/v1alpha1"
	"github.com/tektoncd/operator/pkg/client/clientset/versioned"
	operatorclient "github.com/tektoncd/operator/pkg/client/injection/client"
	"github.com/tektoncd/operator/pkg/reconciler/common"
	"github.com/tektoncd/operator/pkg/reconciler/kubernetes/tektoninstallerset"
	occommon "github.com/tektoncd/operator/pkg/reconciler/openshift/common"
	"go.uber.org/zap"
	"knative.dev/pkg/injection"
	"knative.dev/pkg/logging"
	"knative.dev/pkg/ptr"
)

const (
	// DefaultDisableAffinityAssistant is default value of disable affinity assistant flag
	DefaultDisableAffinityAssistant = true
	monitoringLabel                 = "openshift.io/cluster-monitoring=true"

	enableMetricsKey          = "enableMetrics"
	enableMetricsDefaultValue = "true"

	versionKey = "VERSION"

	prePipelineInstallerSet  = "PrePipeline"
	postPipelineInstallerSet = "PostPipeline"
)

var (
	preReconcileSelector = fmt.Sprintf("%s=%s,%s=%s",
		tektoninstallerset.InstallerSetType, prePipelineInstallerSet,
		tektoninstallerset.CreatedByKey, createdByValue,
	)
	postReconcileSelector = fmt.Sprintf("%s=%s,%s=%s",
		tektoninstallerset.InstallerSetType, prePipelineInstallerSet,
		tektoninstallerset.CreatedByKey, createdByValue,
	)
)

func OpenShiftExtension(ctx context.Context) common.Extension {
	logger := logging.FromContext(ctx)
	mfclient, err := mfc.NewClient(injection.GetConfig(ctx))
	if err != nil {
		logger.Fatalw("error creating client from injected config", zap.Error(err))
	}
	mflogger := zapr.NewLogger(logger.Named("manifestival").Desugar())
	manifest, err := mf.ManifestFrom(mf.Slice{}, mf.UseClient(mfclient), mf.UseLogger(mflogger))
	if err != nil {
		logger.Fatalw("error creating initial manifest", zap.Error(err))
	}

	version := os.Getenv(versionKey)
	if version == "" {
		logger.Fatal("Failed to find version from env")
	}

	ext := openshiftExtension{
		operatorClientSet: operatorclient.Get(ctx),
		manifest:          manifest,
		version:           version,
	}
	return ext
}

type openshiftExtension struct {
	operatorClientSet versioned.Interface
	manifest          mf.Manifest
	version           string
}

func (oe openshiftExtension) Transformers(comp v1alpha1.TektonComponent) []mf.Transformer {
	trns := []mf.Transformer{
		occommon.ApplyCABundles,
		occommon.RemoveRunAsUser(),
	}

	pipeline := comp.(*v1alpha1.TektonPipeline)

	// Add monitoring label if metrics is enabled
	value := findParam(pipeline.Spec.Params, enableMetricsKey)
	if value == "" || value == "true" {
		trns = append(trns, common.InjectLabelOnNamespace(monitoringLabel))
	}

	return trns
}
func (oe openshiftExtension) PreReconcile(ctx context.Context, comp v1alpha1.TektonComponent) error {
	koDataDir := os.Getenv(common.KoEnvKey)
	tp := comp.(*v1alpha1.TektonPipeline)

	SetDefault(&tp.Spec.Pipeline)

	exist, err := checkIfInstallerSetExist(ctx, oe.operatorClientSet, oe.version, tp, preReconcileSelector)
	if err != nil {
		return err
	}

	// If installer set doesn't exist then create a new one
	if !exist {

		// make sure that openshift-pipelines namespace exists
		namespaceLocation := filepath.Join(koDataDir, "tekton-namespace")
		if err := common.AppendManifest(&oe.manifest, namespaceLocation); err != nil {
			return err
		}

		// add inject CA bundles manifests
		cabundlesLocation := filepath.Join(koDataDir, "cabundles")
		if err := common.AppendManifest(&oe.manifest, cabundlesLocation); err != nil {
			return err
		}

		// add pipelines-scc
		pipelinesSCCLocation := filepath.Join(koDataDir, "tekton-pipeline", "00-prereconcile")
		if err := common.AppendManifest(&oe.manifest, pipelinesSCCLocation); err != nil {
			return err
		}

		if err := common.Transform(ctx, &oe.manifest, tp); err != nil {
			return err
		}

		if err := createInstallerSet(ctx, oe.operatorClientSet, tp, oe.manifest, oe.version,
			prePipelineInstallerSet); err != nil {
			return err
		}
	}

	return nil
}

func (oe openshiftExtension) PostReconcile(ctx context.Context, comp v1alpha1.TektonComponent) error {
	koDataDir := os.Getenv(common.KoEnvKey)
	pipeline := comp.(*v1alpha1.TektonPipeline)

	// Install monitoring if metrics is enabled
	value := findParam(pipeline.Spec.Params, enableMetricsKey)

	if value == "true" {
		exist, err := checkIfInstallerSetExist(ctx, oe.operatorClientSet, oe.version, pipeline, postReconcileSelector)
		if err != nil {
			return err
		}

		if !exist {

			monitoringLocation := filepath.Join(koDataDir, "openshift-monitoring")
			if err := common.AppendManifest(&oe.manifest, monitoringLocation); err != nil {
				return err
			}

			if err := common.Transform(ctx, &oe.manifest, pipeline); err != nil {
				return err
			}

			if err := createInstallerSet(ctx, oe.operatorClientSet, pipeline, oe.manifest, oe.version,
				postPipelineInstallerSet); err != nil {
				return err
			}
		}

	} else {
		return deleteInstallerSet(ctx, oe.operatorClientSet, pipeline, postPipelineInstallerSet, postReconcileSelector)
	}

	return nil
}
func (oe openshiftExtension) Finalize(ctx context.Context, comp v1alpha1.TektonComponent) error {
	pipeline := comp.(*v1alpha1.TektonPipeline)
	if err := deleteInstallerSet(ctx, oe.operatorClientSet, pipeline,
		postPipelineInstallerSet, postReconcileSelector); err != nil {
		return err
	}
	if err := deleteInstallerSet(ctx, oe.operatorClientSet, pipeline,
		postPipelineInstallerSet, postReconcileSelector); err != nil {
		return err
	}
	return nil
}

func SetDefault(pipeline *v1alpha1.Pipeline) {

	// Set default service account as pipeline
	if pipeline.DefaultServiceAccount == "" {
		pipeline.DefaultServiceAccount = common.DefaultSA
	}

	// Set `disable-affinity-assistant` to true if not set in CR
	// webhook will not set any value but by default in pipelines configmap it will be false
	if pipeline.DisableAffinityAssistant == nil {
		pipeline.DisableAffinityAssistant = ptr.Bool(DefaultDisableAffinityAssistant)
	}

	// Add params with default values if not defined by user
	var found = false
	for i, p := range pipeline.Params {
		if p.Name == enableMetricsKey {
			found = true
			// If the value set is invalid then set key to default value
			// Not returning an error if the values is invalid as
			// we validate in reconciler and this would affect the
			// rest of the installation
			if p.Value != "false" && p.Value != "true" {
				pipeline.Params[i].Value = enableMetricsDefaultValue
			}
			break
		}
	}

	if !found {
		pipeline.Params = append(pipeline.Params, v1alpha1.Param{
			Name:  enableMetricsKey,
			Value: enableMetricsDefaultValue,
		})
	}
}

func findParam(params []v1alpha1.Param, param string) string {
	for _, p := range params {
		if p.Name == param {
			return p.Value
		}
	}
	return ""
}
<commit_msg>[OpenShift] fixes post pipelines installerSet creation<commit_after>/*
Copyright 2020 The Tekton Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tektonpipeline

import (
	"context"
	"fmt"
	"os"
	"path/filepath"

	"github.com/go-logr/zapr"
	mfc "github.com/manifestival/client-go-client"
	mf "github.com/manifestival/manifestival"
	"github.com/tektoncd/operator/pkg/apis/operator/v1alpha1"
	"github.com/tektoncd/operator/pkg/client/clientset/versioned"
	operatorclient "github.com/tektoncd/operator/pkg/client/injection/client"
	"github.com/tektoncd/operator/pkg/reconciler/common"
	"github.com/tektoncd/operator/pkg/reconciler/kubernetes/tektoninstallerset"
	occommon "github.com/tektoncd/operator/pkg/reconciler/openshift/common"
	"go.uber.org/zap"
	"knative.dev/pkg/injection"
	"knative.dev/pkg/logging"
	"knative.dev/pkg/ptr"
)

const (
	// DefaultDisableAffinityAssistant is default value of disable affinity assistant flag
	DefaultDisableAffinityAssistant = true
	monitoringLabel                 = "openshift.io/cluster-monitoring=true"

	enableMetricsKey          = "enableMetrics"
	enableMetricsDefaultValue = "true"

	versionKey = "VERSION"

	prePipelineInstallerSet  = "PrePipeline"
	postPipelineInstallerSet = "PostPipeline"
)

var (
	preReconcileSelector = fmt.Sprintf("%s=%s,%s=%s",
		tektoninstallerset.InstallerSetType, prePipelineInstallerSet,
		tektoninstallerset.CreatedByKey, createdByValue,
	)
	postReconcileSelector = fmt.Sprintf("%s=%s,%s=%s",
		tektoninstallerset.InstallerSetType, postPipelineInstallerSet,
		tektoninstallerset.CreatedByKey, createdByValue,
	)
)

func OpenShiftExtension(ctx context.Context) common.Extension {
	logger := logging.FromContext(ctx)
	mfclient, err := mfc.NewClient(injection.GetConfig(ctx))
	if err != nil {
		logger.Fatalw("error creating client from injected config", zap.Error(err))
	}
	mflogger := zapr.NewLogger(logger.Named("manifestival").Desugar())
	manifest, err := mf.ManifestFrom(mf.Slice{}, mf.UseClient(mfclient), mf.UseLogger(mflogger))
	if err != nil {
		logger.Fatalw("error creating initial manifest", zap.Error(err))
	}

	version := os.Getenv(versionKey)
	if version == "" {
		logger.Fatal("Failed to find version from env")
	}

	ext := openshiftExtension{
		operatorClientSet: operatorclient.Get(ctx),
		manifest:          manifest,
		version:           version,
	}
	return ext
}

type openshiftExtension struct {
	operatorClientSet versioned.Interface
	manifest          mf.Manifest
	version           string
}

func (oe openshiftExtension) Transformers(comp v1alpha1.TektonComponent) []mf.Transformer {
	trns := []mf.Transformer{
		occommon.ApplyCABundles,
		occommon.RemoveRunAsUser(),
	}

	pipeline := comp.(*v1alpha1.TektonPipeline)

	// Add monitoring label if metrics is enabled
	value := findParam(pipeline.Spec.Params, enableMetricsKey)
	if value == "" || value == "true" {
		trns = append(trns, common.InjectLabelOnNamespace(monitoringLabel))
	}

	return trns
}
func (oe openshiftExtension) PreReconcile(ctx context.Context, comp v1alpha1.TektonComponent) error {
	koDataDir := os.Getenv(common.KoEnvKey)
	tp := comp.(*v1alpha1.TektonPipeline)

	SetDefault(&tp.Spec.Pipeline)

	exist, err := checkIfInstallerSetExist(ctx, oe.operatorClientSet, oe.version, tp, preReconcileSelector)
	if err != nil {
		return err
	}

	// If installer set doesn't exist then create a new one
	if !exist {

		// make sure that openshift-pipelines namespace exists
		namespaceLocation := filepath.Join(koDataDir, "tekton-namespace")
		if err := common.AppendManifest(&oe.manifest, namespaceLocation); err != nil {
			return err
		}

		// add inject CA bundles manifests
		cabundlesLocation := filepath.Join(koDataDir, "cabundles")
		if err := common.AppendManifest(&oe.manifest, cabundlesLocation); err != nil {
			return err
		}

		// add pipelines-scc
		pipelinesSCCLocation := filepath.Join(koDataDir, "tekton-pipeline", "00-prereconcile")
		if err := common.AppendManifest(&oe.manifest, pipelinesSCCLocation); err != nil {
			return err
		}

		if err := common.Transform(ctx, &oe.manifest, tp); err != nil {
			return err
		}

		if err := createInstallerSet(ctx, oe.operatorClientSet, tp, oe.manifest, oe.version,
			prePipelineInstallerSet); err != nil {
			return err
		}
	}

	return nil
}

func (oe openshiftExtension) PostReconcile(ctx context.Context, comp v1alpha1.TektonComponent) error {
	koDataDir := os.Getenv(common.KoEnvKey)
	pipeline := comp.(*v1alpha1.TektonPipeline)

	// Install monitoring if metrics is enabled
	value := findParam(pipeline.Spec.Params, enableMetricsKey)

	if value == "true" {
		exist, err := checkIfInstallerSetExist(ctx, oe.operatorClientSet, oe.version, pipeline, postReconcileSelector)
		if err != nil {
			return err
		}

		if !exist {

			monitoringLocation := filepath.Join(koDataDir, "openshift-monitoring")
			if err := common.AppendManifest(&oe.manifest, monitoringLocation); err != nil {
				return err
			}

			if err := common.Transform(ctx, &oe.manifest, pipeline); err != nil {
				return err
			}

			if err := createInstallerSet(ctx, oe.operatorClientSet, pipeline, oe.manifest, oe.version,
				postPipelineInstallerSet); err != nil {
				return err
			}
		}

	} else {
		return deleteInstallerSet(ctx, oe.operatorClientSet, pipeline, postPipelineInstallerSet, postReconcileSelector)
	}

	return nil
}
func (oe openshiftExtension) Finalize(ctx context.Context, comp v1alpha1.TektonComponent) error {
	pipeline := comp.(*v1alpha1.TektonPipeline)
	if err := deleteInstallerSet(ctx, oe.operatorClientSet, pipeline,
		postPipelineInstallerSet, postReconcileSelector); err != nil {
		return err
	}
	if err := deleteInstallerSet(ctx, oe.operatorClientSet, pipeline,
		prePipelineInstallerSet, preReconcileSelector); err != nil {
		return err
	}
	return nil
}

func SetDefault(pipeline *v1alpha1.Pipeline) {

	// Set default service account as pipeline
	if pipeline.DefaultServiceAccount == "" {
		pipeline.DefaultServiceAccount = common.DefaultSA
	}

	// Set `disable-affinity-assistant` to true if not set in CR
	// webhook will not set any value but by default in pipelines configmap it will be false
	if pipeline.DisableAffinityAssistant == nil {
		pipeline.DisableAffinityAssistant = ptr.Bool(DefaultDisableAffinityAssistant)
	}

	// Add params with default values if not defined by user
	var found = false
	for i, p := range pipeline.Params {
		if p.Name == enableMetricsKey {
			found = true
			// If the value set is invalid then set key to default value
			// Not returning an error if the values is invalid as
			// we validate in reconciler and this would affect the
			// rest of the installation
			if p.Value != "false" && p.Value != "true" {
				pipeline.Params[i].Value = enableMetricsDefaultValue
			}
			break
		}
	}

	if !found {
		pipeline.Params = append(pipeline.Params, v1alpha1.Param{
			Name:  enableMetricsKey,
			Value: enableMetricsDefaultValue,
		})
	}
}

func findParam(params []v1alpha1.Param, param string) string {
	for _, p := range params {
		if p.Name == param {
			return p.Value
		}
	}
	return ""
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Danilo Bürger <info@danilobuerger.de>

package oauth2

import (
	"encoding"
	"net/url"
)

// AccessResponse holds a valid and authorized access response.
type AccessResponse struct {
	AccessToken  string
	TokenType    string
	ExpiresIn    string
	RefreshToken string
	Info         map[string]interface{}
}

// ToMap converts the access response to a map.
func (r *AccessResponse) ToMap() map[string]interface{} {
	m := r.Info

	m["access_token"] = r.AccessToken
	m["token_type"] = r.TokenType
	m["expires_in"] = r.ExpiresIn

	if r.RefreshToken != "" {
		m["refresh_token"] = r.RefreshToken
	}

	return m
}

// ToValues converts the access response to values.
func (r *AccessResponse) ToValues() url.Values {
	values := url.Values{}
	for k, vi := range r.Info {
		if vs, ok := vi.([]string); ok {
			for _, v := range vs {
				values.Add(k, v)
			}
		} else if v, ok := vi.(string); ok {
			values.Set(k, v)
		} else if v, ok := vi.(encoding.TextMarshaler); ok {
			text, err := v.MarshalText()
			if err == nil {
				values.Set(k, string(text))
			}
		}
	}

	values.Set("access_token", r.AccessToken)
	values.Set("token_type", r.TokenType)
	values.Set("expires_in", r.ExpiresIn)

	return values
}
<commit_msg>Add bool info values to access response<commit_after>// Copyright (c) 2016 Danilo Bürger <info@danilobuerger.de>

package oauth2

import (
	"encoding"
	"net/url"
	"strconv"
)

// AccessResponse holds a valid and authorized access response.
type AccessResponse struct {
	AccessToken  string
	TokenType    string
	ExpiresIn    string
	RefreshToken string
	Info         map[string]interface{}
}

// ToMap converts the access response to a map.
func (r *AccessResponse) ToMap() map[string]interface{} {
	m := r.Info

	m["access_token"] = r.AccessToken
	m["token_type"] = r.TokenType
	m["expires_in"] = r.ExpiresIn

	if r.RefreshToken != "" {
		m["refresh_token"] = r.RefreshToken
	}

	return m
}

// ToValues converts the access response to values.
func (r *AccessResponse) ToValues() url.Values {
	values := url.Values{}
	for k, vi := range r.Info {
		if vs, ok := vi.([]string); ok {
			for _, v := range vs {
				values.Add(k, v)
			}
		} else if v, ok := vi.(string); ok {
			values.Set(k, v)
		} else if v, ok := vi.(bool); ok {
			values.Set(k, strconv.FormatBool(v))
		} else if v, ok := vi.(encoding.TextMarshaler); ok {
			text, err := v.MarshalText()
			if err == nil {
				values.Set(k, string(text))
			}
		}
	}

	values.Set("access_token", r.AccessToken)
	values.Set("token_type", r.TokenType)
	values.Set("expires_in", r.ExpiresIn)

	return values
}
<|endoftext|>


<commit_before>package account

import (
	"context"
	"encoding/json"
	"sync"
	"sync/atomic"
	"time"

	dbm "github.com/tendermint/tmlibs/db"

	"github.com/bytom/errors"
	"github.com/bytom/protocol"
	"github.com/bytom/protocol/bc"
	"github.com/bytom/sync/idempotency"
)

var (
	// ErrInsufficient indicates the account doesn't contain enough
	// units of the requested asset to satisfy the reservation.
	// New units must be deposited into the account in order to
	// satisfy the request; change will not be sufficient.
	ErrInsufficient = errors.New("reservation found insufficient funds")

	// ErrReserved indicates that a reservation could not be
	// satisfied because some of the outputs were already reserved.
	// When those reservations are finalized into a transaction
	// (and no other transaction spends funds from the account),
	// new change outputs will be created
	// in sufficient amounts to satisfy the request.
	ErrReserved = errors.New("reservation found outputs already reserved")
	// ErrMatchUTXO indicates the account doesn't contain enough utxo to satisfy the reservation.
	ErrMatchUTXO = errors.New("can't match enough valid utxos")
	// ErrReservation indicates the reserver doesn't found the reservation with the provided ID.
	ErrReservation = errors.New("couldn't find reservation")
)

// UTXO describes an individual account utxo.
type UTXO struct {
	OutputID bc.Hash
	SourceID bc.Hash

	// Avoiding AssetAmount here so that new(utxo) doesn't produce an
	// AssetAmount with a nil AssetId.
	AssetID bc.AssetID
	Amount  uint64

	SourcePos      uint64
	ControlProgram []byte

	AccountID           string
	Address             string
	ControlProgramIndex uint64
	ValidHeight         uint64
	Change              bool
}

func (u *UTXO) source() source {
	return source{AssetID: u.AssetID, AccountID: u.AccountID}
}

// source describes the criteria to use when selecting UTXOs.
type source struct {
	AssetID   bc.AssetID
	AccountID string
}

// reservation describes a reservation of a set of UTXOs belonging
// to a particular account. Reservations are immutable.
type reservation struct {
	ID          uint64
	Source      source
	UTXOs       []*UTXO
	Change      uint64
	Expiry      time.Time
	ClientToken *string
}

func newReserver(c *protocol.Chain, walletdb dbm.DB) *reserver {
	return &reserver{
		c:            c,
		db:           walletdb,
		reservations: make(map[uint64]*reservation),
		sources:      make(map[source]*sourceReserver),
	}
}

// reserver implements a utxo reserver that stores reservations
// in-memory. It relies on the account_utxos table for the source of
// truth of valid UTXOs but tracks which of those UTXOs are reserved
// in-memory.
//
// To reduce latency and prevent deadlock, no two mutexes (either on
// reserver or sourceReserver) should be held at the same time
//
// reserver ensures idempotency of reservations until the reservation
// expiration.
type reserver struct {
	c                 *protocol.Chain
	db                dbm.DB
	nextReservationID uint64
	idempotency       idempotency.Group

	reservationsMu sync.Mutex
	reservations   map[uint64]*reservation

	sourcesMu sync.Mutex
	sources   map[source]*sourceReserver
}

// Reserve selects and reserves UTXOs according to the criteria provided
// in source. The resulting reservation expires at exp.
func (re *reserver) Reserve(src source, amount uint64, clientToken *string, exp time.Time) (*reservation, error) {

	if clientToken == nil {
		return re.reserve(src, amount, clientToken, exp)
	}

	untypedRes, err := re.idempotency.Once(*clientToken, func() (interface{}, error) {
		return re.reserve(src, amount, clientToken, exp)
	})
	return untypedRes.(*reservation), err
}

func (re *reserver) reserve(src source, amount uint64, clientToken *string, exp time.Time) (res *reservation, err error) {
	sourceReserver := re.source(src)

	// Try to reserve the right amount.
	rid := atomic.AddUint64(&re.nextReservationID, 1)
	reserved, total, isImmature, err := sourceReserver.reserve(rid, amount)
	if err != nil {
		if isImmature {
			return nil, errors.WithDetail(err, "some coinbase utxos are immature")
		}
		return nil, err
	}

	res = &reservation{
		ID:          rid,
		Source:      src,
		UTXOs:       reserved,
		Expiry:      exp,
		ClientToken: clientToken,
	}

	// Save the successful reservation.
	re.reservationsMu.Lock()
	defer re.reservationsMu.Unlock()
	re.reservations[rid] = res

	// Make change if necessary
	if total > amount {
		res.Change = total - amount
	}
	return res, nil
}

// ReserveUTXO reserves a specific utxo for spending. The resulting
// reservation expires at exp.
func (re *reserver) ReserveUTXO(ctx context.Context, out bc.Hash, clientToken *string, exp time.Time) (*reservation, error) {
	if clientToken == nil {
		return re.reserveUTXO(ctx, out, exp, nil)
	}

	untypedRes, err := re.idempotency.Once(*clientToken, func() (interface{}, error) {
		return re.reserveUTXO(ctx, out, exp, clientToken)
	})
	return untypedRes.(*reservation), err
}

func (re *reserver) reserveUTXO(ctx context.Context, out bc.Hash, exp time.Time, clientToken *string) (*reservation, error) {
	u, err := findSpecificUTXO(re.db, out)
	if err != nil {
		return nil, err
	}

	//u.ValidHeight > 0 means coinbase utxo
	if u.ValidHeight > 0 && u.ValidHeight > re.c.BestBlockHeight() {
		return nil, errors.WithDetail(ErrMatchUTXO, "this coinbase utxo is immature")
	}

	rid := atomic.AddUint64(&re.nextReservationID, 1)
	err = re.source(u.source()).reserveUTXO(rid, u)
	if err != nil {
		return nil, err
	}

	res := &reservation{
		ID:          rid,
		Source:      u.source(),
		UTXOs:       []*UTXO{u},
		Expiry:      exp,
		ClientToken: clientToken,
	}
	re.reservationsMu.Lock()
	re.reservations[rid] = res
	re.reservationsMu.Unlock()
	return res, nil
}

// Cancel makes a best-effort attempt at canceling the reservation with
// the provided ID.
func (re *reserver) Cancel(ctx context.Context, rid uint64) error {
	re.reservationsMu.Lock()
	res, ok := re.reservations[rid]
	delete(re.reservations, rid)
	re.reservationsMu.Unlock()
	if !ok {
		return errors.Wrapf(ErrReservation, "rid=%d", rid)
	}
	re.source(res.Source).cancel(res)
	/*if res.ClientToken != nil {
		re.idempotency.Forget(*res.ClientToken)
	}*/
	return nil
}

// ExpireReservations cleans up all reservations that have expired,
// making their UTXOs available for reservation again.
func (re *reserver) ExpireReservations(ctx context.Context) error {
	// Remove records of any reservations that have expired.
	now := time.Now()
	var canceled []*reservation
	re.reservationsMu.Lock()
	for rid, res := range re.reservations {
		if res.Expiry.Before(now) {
			canceled = append(canceled, res)
			delete(re.reservations, rid)
		}
	}
	re.reservationsMu.Unlock()

	// If we removed any expired reservations, update the corresponding
	// source reservers.
	for _, res := range canceled {
		re.source(res.Source).cancel(res)
		/*if res.ClientToken != nil {
			re.idempotency.Forget(*res.ClientToken)
		}*/
	}

	// TODO(jackson): Cleanup any source reservers that don't have
	// anything reserved. It'll be a little tricky because of our
	// locking scheme.
	return nil
}

func (re *reserver) source(src source) *sourceReserver {
	re.sourcesMu.Lock()
	defer re.sourcesMu.Unlock()

	sr, ok := re.sources[src]
	if ok {
		return sr
	}

	sr = &sourceReserver{
		db:            re.db,
		src:           src,
		reserved:      make(map[bc.Hash]uint64),
		currentHeight: re.c.BestBlockHeight,
	}
	re.sources[src] = sr
	return sr
}

type sourceReserver struct {
	db            dbm.DB
	src           source
	currentHeight func() uint64
	mu            sync.Mutex
	reserved      map[bc.Hash]uint64
}

func (sr *sourceReserver) reserve(rid uint64, amount uint64) ([]*UTXO, uint64, bool, error) {
	var (
		reserved, unavailable uint64
		reservedUTXOs         []*UTXO
	)

	utxos, isImmature, err := findMatchingUTXOs(sr.db, sr.src, sr.currentHeight)
	if err != nil {
		return nil, 0, isImmature, errors.Wrap(err)
	}

	sr.mu.Lock()
	defer sr.mu.Unlock()
	for _, u := range utxos {
		// If the UTXO is already reserved, skip it.
		if _, ok := sr.reserved[u.OutputID]; ok {
			unavailable += u.Amount
			continue
		}

		reserved += u.Amount
		reservedUTXOs = append(reservedUTXOs, u)
		if reserved >= amount {
			break
		}
	}
	if reserved+unavailable < amount {
		// Even if everything was available, this account wouldn't have
		// enough to satisfy the request.
		return nil, 0, isImmature, ErrInsufficient
	}
	if reserved < amount {
		// The account has enough for the request, but some is tied up in
		// other reservations.
		return nil, 0, isImmature, ErrReserved
	}

	// We've found enough to satisfy the request.
	for _, u := range reservedUTXOs {
		sr.reserved[u.OutputID] = rid
	}

	return reservedUTXOs, reserved, isImmature, nil
}

func (sr *sourceReserver) reserveUTXO(rid uint64, utxo *UTXO) error {
	sr.mu.Lock()
	defer sr.mu.Unlock()

	_, isReserved := sr.reserved[utxo.OutputID]
	if isReserved {
		return ErrReserved
	}

	sr.reserved[utxo.OutputID] = rid
	return nil
}

func (sr *sourceReserver) cancel(res *reservation) {
	sr.mu.Lock()
	defer sr.mu.Unlock()
	for _, utxo := range res.UTXOs {
		delete(sr.reserved, utxo.OutputID)
	}
}

func findMatchingUTXOs(db dbm.DB, src source, currentHeight func() uint64) ([]*UTXO, bool, error) {
	utxos := []*UTXO{}
	isImmature := false
	utxoIter := db.IteratorPrefix([]byte(UTXOPreFix))
	defer utxoIter.Release()

	for utxoIter.Next() {
		u := &UTXO{}
		if err := json.Unmarshal(utxoIter.Value(), u); err != nil {
			return nil, false, errors.Wrap(err)
		}

		//u.ValidHeight > 0 means coinbase utxo
		if u.ValidHeight > 0 && u.ValidHeight > currentHeight() {
			isImmature = true
			continue
		}

		if u.AccountID == src.AccountID && u.AssetID == src.AssetID {
			utxos = append(utxos, u)
		}
	}

	if len(utxos) == 0 {
		return nil, isImmature, ErrMatchUTXO
	}
	return utxos, isImmature, nil
}

func findSpecificUTXO(db dbm.DB, outHash bc.Hash) (*UTXO, error) {
	u := &UTXO{}

	data := db.Get(StandardUTXOKey(outHash))
	if data == nil {
		if data = db.Get(ContractUTXOKey(outHash)); data == nil {
			return nil, errors.Wrapf(ErrMatchUTXO, "output_id = %s", outHash.String())
		}
	}
	return u, json.Unmarshal(data, u)
}
<commit_msg>fix x86-32 exeception for 64-bit aligned<commit_after>package account

import (
	"context"
	"encoding/json"
	"sync"
	"sync/atomic"
	"time"

	dbm "github.com/tendermint/tmlibs/db"

	"github.com/bytom/errors"
	"github.com/bytom/protocol"
	"github.com/bytom/protocol/bc"
	"github.com/bytom/sync/idempotency"
)

var (
	// ErrInsufficient indicates the account doesn't contain enough
	// units of the requested asset to satisfy the reservation.
	// New units must be deposited into the account in order to
	// satisfy the request; change will not be sufficient.
	ErrInsufficient = errors.New("reservation found insufficient funds")

	// ErrReserved indicates that a reservation could not be
	// satisfied because some of the outputs were already reserved.
	// When those reservations are finalized into a transaction
	// (and no other transaction spends funds from the account),
	// new change outputs will be created
	// in sufficient amounts to satisfy the request.
	ErrReserved = errors.New("reservation found outputs already reserved")
	// ErrMatchUTXO indicates the account doesn't contain enough utxo to satisfy the reservation.
	ErrMatchUTXO = errors.New("can't match enough valid utxos")
	// ErrReservation indicates the reserver doesn't found the reservation with the provided ID.
	ErrReservation = errors.New("couldn't find reservation")
)

// UTXO describes an individual account utxo.
type UTXO struct {
	OutputID bc.Hash
	SourceID bc.Hash

	// Avoiding AssetAmount here so that new(utxo) doesn't produce an
	// AssetAmount with a nil AssetId.
	AssetID bc.AssetID
	Amount  uint64

	SourcePos      uint64
	ControlProgram []byte

	AccountID           string
	Address             string
	ControlProgramIndex uint64
	ValidHeight         uint64
	Change              bool
}

func (u *UTXO) source() source {
	return source{AssetID: u.AssetID, AccountID: u.AccountID}
}

// source describes the criteria to use when selecting UTXOs.
type source struct {
	AssetID   bc.AssetID
	AccountID string
}

// reservation describes a reservation of a set of UTXOs belonging
// to a particular account. Reservations are immutable.
type reservation struct {
	ID          uint64
	Source      source
	UTXOs       []*UTXO
	Change      uint64
	Expiry      time.Time
	ClientToken *string
}

func newReserver(c *protocol.Chain, walletdb dbm.DB) *reserver {
	return &reserver{
		c:            c,
		db:           walletdb,
		reservations: make(map[uint64]*reservation),
		sources:      make(map[source]*sourceReserver),
	}
}

// reserver implements a utxo reserver that stores reservations
// in-memory. It relies on the account_utxos table for the source of
// truth of valid UTXOs but tracks which of those UTXOs are reserved
// in-memory.
//
// To reduce latency and prevent deadlock, no two mutexes (either on
// reserver or sourceReserver) should be held at the same time
//
// reserver ensures idempotency of reservations until the reservation
// expiration.
type reserver struct {
	// `sync/atomic` expects the first word in an allocated struct to be 64-bit
	// aligned on both ARM and x86-32. See https://goo.gl/zW7dgq for more details.
	nextReservationID uint64
	c                 *protocol.Chain
	db                dbm.DB
	idempotency       idempotency.Group

	reservationsMu sync.Mutex
	reservations   map[uint64]*reservation

	sourcesMu sync.Mutex
	sources   map[source]*sourceReserver
}

// Reserve selects and reserves UTXOs according to the criteria provided
// in source. The resulting reservation expires at exp.
func (re *reserver) Reserve(src source, amount uint64, clientToken *string, exp time.Time) (*reservation, error) {

	if clientToken == nil {
		return re.reserve(src, amount, clientToken, exp)
	}

	untypedRes, err := re.idempotency.Once(*clientToken, func() (interface{}, error) {
		return re.reserve(src, amount, clientToken, exp)
	})
	return untypedRes.(*reservation), err
}

func (re *reserver) reserve(src source, amount uint64, clientToken *string, exp time.Time) (res *reservation, err error) {
	sourceReserver := re.source(src)

	// Try to reserve the right amount.
	rid := atomic.AddUint64(&re.nextReservationID, 1)
	reserved, total, isImmature, err := sourceReserver.reserve(rid, amount)
	if err != nil {
		if isImmature {
			return nil, errors.WithDetail(err, "some coinbase utxos are immature")
		}
		return nil, err
	}

	res = &reservation{
		ID:          rid,
		Source:      src,
		UTXOs:       reserved,
		Expiry:      exp,
		ClientToken: clientToken,
	}

	// Save the successful reservation.
	re.reservationsMu.Lock()
	defer re.reservationsMu.Unlock()
	re.reservations[rid] = res

	// Make change if necessary
	if total > amount {
		res.Change = total - amount
	}
	return res, nil
}

// ReserveUTXO reserves a specific utxo for spending. The resulting
// reservation expires at exp.
func (re *reserver) ReserveUTXO(ctx context.Context, out bc.Hash, clientToken *string, exp time.Time) (*reservation, error) {
	if clientToken == nil {
		return re.reserveUTXO(ctx, out, exp, nil)
	}

	untypedRes, err := re.idempotency.Once(*clientToken, func() (interface{}, error) {
		return re.reserveUTXO(ctx, out, exp, clientToken)
	})
	return untypedRes.(*reservation), err
}

func (re *reserver) reserveUTXO(ctx context.Context, out bc.Hash, exp time.Time, clientToken *string) (*reservation, error) {
	u, err := findSpecificUTXO(re.db, out)
	if err != nil {
		return nil, err
	}

	//u.ValidHeight > 0 means coinbase utxo
	if u.ValidHeight > 0 && u.ValidHeight > re.c.BestBlockHeight() {
		return nil, errors.WithDetail(ErrMatchUTXO, "this coinbase utxo is immature")
	}

	rid := atomic.AddUint64(&re.nextReservationID, 1)
	err = re.source(u.source()).reserveUTXO(rid, u)
	if err != nil {
		return nil, err
	}

	res := &reservation{
		ID:          rid,
		Source:      u.source(),
		UTXOs:       []*UTXO{u},
		Expiry:      exp,
		ClientToken: clientToken,
	}
	re.reservationsMu.Lock()
	re.reservations[rid] = res
	re.reservationsMu.Unlock()
	return res, nil
}

// Cancel makes a best-effort attempt at canceling the reservation with
// the provided ID.
func (re *reserver) Cancel(ctx context.Context, rid uint64) error {
	re.reservationsMu.Lock()
	res, ok := re.reservations[rid]
	delete(re.reservations, rid)
	re.reservationsMu.Unlock()
	if !ok {
		return errors.Wrapf(ErrReservation, "rid=%d", rid)
	}
	re.source(res.Source).cancel(res)
	/*if res.ClientToken != nil {
		re.idempotency.Forget(*res.ClientToken)
	}*/
	return nil
}

// ExpireReservations cleans up all reservations that have expired,
// making their UTXOs available for reservation again.
func (re *reserver) ExpireReservations(ctx context.Context) error {
	// Remove records of any reservations that have expired.
	now := time.Now()
	var canceled []*reservation
	re.reservationsMu.Lock()
	for rid, res := range re.reservations {
		if res.Expiry.Before(now) {
			canceled = append(canceled, res)
			delete(re.reservations, rid)
		}
	}
	re.reservationsMu.Unlock()

	// If we removed any expired reservations, update the corresponding
	// source reservers.
	for _, res := range canceled {
		re.source(res.Source).cancel(res)
		/*if res.ClientToken != nil {
			re.idempotency.Forget(*res.ClientToken)
		}*/
	}

	// TODO(jackson): Cleanup any source reservers that don't have
	// anything reserved. It'll be a little tricky because of our
	// locking scheme.
	return nil
}

func (re *reserver) source(src source) *sourceReserver {
	re.sourcesMu.Lock()
	defer re.sourcesMu.Unlock()

	sr, ok := re.sources[src]
	if ok {
		return sr
	}

	sr = &sourceReserver{
		db:            re.db,
		src:           src,
		reserved:      make(map[bc.Hash]uint64),
		currentHeight: re.c.BestBlockHeight,
	}
	re.sources[src] = sr
	return sr
}

type sourceReserver struct {
	db            dbm.DB
	src           source
	currentHeight func() uint64
	mu            sync.Mutex
	reserved      map[bc.Hash]uint64
}

func (sr *sourceReserver) reserve(rid uint64, amount uint64) ([]*UTXO, uint64, bool, error) {
	var (
		reserved, unavailable uint64
		reservedUTXOs         []*UTXO
	)

	utxos, isImmature, err := findMatchingUTXOs(sr.db, sr.src, sr.currentHeight)
	if err != nil {
		return nil, 0, isImmature, errors.Wrap(err)
	}

	sr.mu.Lock()
	defer sr.mu.Unlock()
	for _, u := range utxos {
		// If the UTXO is already reserved, skip it.
		if _, ok := sr.reserved[u.OutputID]; ok {
			unavailable += u.Amount
			continue
		}

		reserved += u.Amount
		reservedUTXOs = append(reservedUTXOs, u)
		if reserved >= amount {
			break
		}
	}
	if reserved+unavailable < amount {
		// Even if everything was available, this account wouldn't have
		// enough to satisfy the request.
		return nil, 0, isImmature, ErrInsufficient
	}
	if reserved < amount {
		// The account has enough for the request, but some is tied up in
		// other reservations.
		return nil, 0, isImmature, ErrReserved
	}

	// We've found enough to satisfy the request.
	for _, u := range reservedUTXOs {
		sr.reserved[u.OutputID] = rid
	}

	return reservedUTXOs, reserved, isImmature, nil
}

func (sr *sourceReserver) reserveUTXO(rid uint64, utxo *UTXO) error {
	sr.mu.Lock()
	defer sr.mu.Unlock()

	_, isReserved := sr.reserved[utxo.OutputID]
	if isReserved {
		return ErrReserved
	}

	sr.reserved[utxo.OutputID] = rid
	return nil
}

func (sr *sourceReserver) cancel(res *reservation) {
	sr.mu.Lock()
	defer sr.mu.Unlock()
	for _, utxo := range res.UTXOs {
		delete(sr.reserved, utxo.OutputID)
	}
}

func findMatchingUTXOs(db dbm.DB, src source, currentHeight func() uint64) ([]*UTXO, bool, error) {
	utxos := []*UTXO{}
	isImmature := false
	utxoIter := db.IteratorPrefix([]byte(UTXOPreFix))
	defer utxoIter.Release()

	for utxoIter.Next() {
		u := &UTXO{}
		if err := json.Unmarshal(utxoIter.Value(), u); err != nil {
			return nil, false, errors.Wrap(err)
		}

		//u.ValidHeight > 0 means coinbase utxo
		if u.ValidHeight > 0 && u.ValidHeight > currentHeight() {
			isImmature = true
			continue
		}

		if u.AccountID == src.AccountID && u.AssetID == src.AssetID {
			utxos = append(utxos, u)
		}
	}

	if len(utxos) == 0 {
		return nil, isImmature, ErrMatchUTXO
	}
	return utxos, isImmature, nil
}

func findSpecificUTXO(db dbm.DB, outHash bc.Hash) (*UTXO, error) {
	u := &UTXO{}

	data := db.Get(StandardUTXOKey(outHash))
	if data == nil {
		if data = db.Get(ContractUTXOKey(outHash)); data == nil {
			return nil, errors.Wrapf(ErrMatchUTXO, "output_id = %s", outHash.String())
		}
	}
	return u, json.Unmarshal(data, u)
}
<|endoftext|>


<commit_before>package decoder

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
)

var (
	// ErrContentTypeUndefined is returned when the request does not include the
	// Content-Type header.
	ErrContentTypeUndefined = fmt.Errorf("Content-Type is undefined")

	// ErrDecoderNotImplemented is returned if the Content-Type does not match
	// one of the defined decoders, i.e.
	//    "application/json" => jsonDecode
	//    "application/xml" => undefined and this error is return
	ErrDecoderNotImplemented = fmt.Errorf("Decoding is not yet implement")
)

// Decode will ready the body of the HTTP request and attempt to unmarshall the
// content into the supplied interface. If the content-type of the request is
// not one that matches a known decoder, then an error will be thrown
func Decode(req *http.Request, v interface{}) error {
	contentType := getContentType(req)
	switch contentType {
	case "application/json":
		return jsonDecode(req, v)
	case "":
		return ErrContentTypeUndefined
	default:
		return ErrDecoderNotImplemented
	}
}

func getContentType(req *http.Request) (contentType string) {
	contentType = strings.TrimSpace(req.Header.Get("Content-Type"))

	return
}

func jsonDecode(req *http.Request, v interface{}) error {
	defer req.Body.Close()

	return json.NewDecoder(req.Body).Decode(&v)
}
<commit_msg>Fixed Content-Type header handling<commit_after>package decoder

import (
	"encoding/json"
	"fmt"
	"net/http"
	"mime"
)

var (
	// ErrContentTypeUndefined is returned when the request does not include the
	// Content-Type header.
	ErrContentTypeUndefined = fmt.Errorf("Content-Type is undefined")

	// ErrDecoderNotImplemented is returned if the Content-Type does not match
	// one of the defined decoders, i.e.
	//    "application/json" => jsonDecode
	//    "application/xml" => undefined and this error is return
	ErrDecoderNotImplemented = fmt.Errorf("Decoding is not yet implement")
)

// Decode will ready the body of the HTTP request and attempt to unmarshall the
// content into the supplied interface. If the content-type of the request is
// not one that matches a known decoder, then an error will be thrown
func Decode(req *http.Request, v interface{}) error {
	contentType, err := getContentType(req)
	if err != nil {
		return err
	}

	switch contentType {
	case "application/json":
		return jsonDecode(req, v)
	case "":
		return ErrContentTypeUndefined
	default:
		return ErrDecoderNotImplemented
	}
}

func getContentType(req *http.Request) (contentType string, err error) {
	contentType, _, err = mime.ParseMediaType(req.Header.Get("Content-Type"))
	return
}

func jsonDecode(req *http.Request, v interface{}) error {
	defer req.Body.Close()

	return json.NewDecoder(req.Body).Decode(&v)
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"github.com/jacobsa/aws/s3"
	"github.com/jacobsa/aws/sdb"
	"github.com/jacobsa/comeback/backup"
	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/config"
	"github.com/jacobsa/comeback/crypto"
	"github.com/jacobsa/comeback/fs"
	s3_kv "github.com/jacobsa/comeback/kv/s3"
	"github.com/jacobsa/comeback/sys"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"time"
)

var g_configFile = flag.String("config", "", "Path to config file.")
var g_jobName = flag.String("job", "", "Job name within the config file.")

func randUint64(randSrc *rand.Rand) uint64

func main() {
	var err error
	flag.Parse()

	// Attempt to read the user's config data.
	if *g_configFile == "" {
		fmt.Println("You must set -config.")
		os.Exit(1)
	}

	configData, err := ioutil.ReadFile(*g_configFile)
	if err != nil {
		fmt.Println("Error reading config file:", err)
		os.Exit(1)
	}

	// Parse the config file.
	cfg, err := config.Parse(configData)
	if err != nil {
		fmt.Println("Parsing config file:", err)
		os.Exit(1)
	}

	// Validate the config file.
	if err := config.Validate(cfg); err != nil {
		fmt.Printf("Config file invalid: %v", err)
		os.Exit(1)
	}

	// Look for the specified job.
	if *g_jobName == "" {
		fmt.Println("You must set -job.")
		os.Exit(1)
	}

	job, ok := cfg.Jobs[*g_jobName]
	if !ok {
		fmt.Println("Unknown job:", *g_jobName)
		os.Exit(1)
	}

	// Create a user registry.
	userRegistry, err := sys.NewUserRegistry()
	if err != nil {
		log.Fatalf("Creating user registry: %v", err)
	}

	// Create a group registry.
	groupRegistry, err := sys.NewGroupRegistry()
	if err != nil {
		log.Fatalf("Creating group registry: %v", err)
	}

	// Create a file system.
	fileSystem, err := fs.NewFileSystem(userRegistry, groupRegistry)
	if err != nil {
		log.Fatalf("Creating file system: %v", err)
	}

	// Open a connection to SimpleDB.
	db, err := sdb.NewSimpleDB(cfg.SdbRegion, cfg.AccessKey)
	if err != nil {
		log.Fatalf("Creating SimpleDB: %v", err)
	}

	// Open a connection to S3.
	bucket, err := s3.OpenBucket(cfg.S3Bucket, cfg.S3Region, cfg.AccessKey)
	if err != nil {
		log.Fatalf("Creating bucket: %v", err)
	}

	// Create the crypter.
	crypter, err := crypto.NewCrypter(cryptoKey)
	if err != nil {
		log.Fatalf("Creating crypter: %v", err)
	}

	// Create the backup registry.
	randSrc := rand.New(rand.NewSource(time.Now().UnixNano()))
	registry, err := backup.NewRegistry(db, cfg.SdbDomain, crypter, randSrc)
	if err != nil {
		log.Fatalf("Creating registry: %v", err)
	}

	// Create the kv store.
	kvStore, err := s3_kv.NewS3KvStore(bucket)
	if err != nil {
		log.Fatalf("Creating kv store: %v", err)
	}

	// Create the blob store.
	blobStore := blob.NewKvBasedBlobStore(kvStore)
	blobStore = blob.NewCheckingStore(blobStore)
	blobStore = blob.NewEncryptingStore(crypter, blobStore)

	// Create the file saver.
	fileSaver, err := backup.NewFileSaver(blobStore, 1<<24)
	if err != nil {
		log.Fatalf("Creating file saver: %v", err)
	}

	// Create a directory saver.
	dirSaver, err := backup.NewDirectorySaver(
		blobStore,
		fileSystem,
		fileSaver)

	if err != nil {
		log.Fatalf("Creating directory saver: %v", err)
	}

	// Run the job.
	score, err := dirSaver.Save(job.BasePath, "", job.Excludes)
	if err != nil {
		log.Fatalf("Saving: %v", err)
	}

	// Register the successful backup.
	completedJob := backup.CompletedJob{
		Id:        randUint64(randSrc),
		Name:      *g_jobName,
		StartTime: startTime,
		Score:     score,
	}

	if err = registry.RecordBackup(completedJob); err != nil {
		log.Fatalf("Recoding to registry: %v", err)
	}

	fmt.Printf("Successfully backed up. ID: %16x\n", completedJob.Id)
}
<commit_msg>Fixed an error.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"github.com/jacobsa/aws/s3"
	"github.com/jacobsa/aws/sdb"
	"github.com/jacobsa/comeback/backup"
	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/config"
	"github.com/jacobsa/comeback/crypto"
	"github.com/jacobsa/comeback/fs"
	s3_kv "github.com/jacobsa/comeback/kv/s3"
	"github.com/jacobsa/comeback/sys"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"time"
)

var g_configFile = flag.String("config", "", "Path to config file.")
var g_jobName = flag.String("job", "", "Job name within the config file.")

func randUint64(randSrc *rand.Rand) uint64

func main() {
	var err error
	flag.Parse()

	// Attempt to read the user's config data.
	if *g_configFile == "" {
		fmt.Println("You must set -config.")
		os.Exit(1)
	}

	configData, err := ioutil.ReadFile(*g_configFile)
	if err != nil {
		fmt.Println("Error reading config file:", err)
		os.Exit(1)
	}

	// Parse the config file.
	cfg, err := config.Parse(configData)
	if err != nil {
		fmt.Println("Parsing config file:", err)
		os.Exit(1)
	}

	// Validate the config file.
	if err := config.Validate(cfg); err != nil {
		fmt.Printf("Config file invalid: %v", err)
		os.Exit(1)
	}

	// Look for the specified job.
	if *g_jobName == "" {
		fmt.Println("You must set -job.")
		os.Exit(1)
	}

	job, ok := cfg.Jobs[*g_jobName]
	if !ok {
		fmt.Println("Unknown job:", *g_jobName)
		os.Exit(1)
	}

	// Create a user registry.
	userRegistry, err := sys.NewUserRegistry()
	if err != nil {
		log.Fatalf("Creating user registry: %v", err)
	}

	// Create a group registry.
	groupRegistry, err := sys.NewGroupRegistry()
	if err != nil {
		log.Fatalf("Creating group registry: %v", err)
	}

	// Create a file system.
	fileSystem, err := fs.NewFileSystem(userRegistry, groupRegistry)
	if err != nil {
		log.Fatalf("Creating file system: %v", err)
	}

	// Open a connection to SimpleDB.
	db, err := sdb.NewSimpleDB(cfg.SdbRegion, cfg.AccessKey)
	if err != nil {
		log.Fatalf("Creating SimpleDB: %v", err)
	}

	// Open a connection to S3.
	bucket, err := s3.OpenBucket(cfg.S3Bucket, cfg.S3Region, cfg.AccessKey)
	if err != nil {
		log.Fatalf("Creating bucket: %v", err)
	}

	// Create the crypter.
	crypter, err := crypto.NewCrypter(cryptoKey)
	if err != nil {
		log.Fatalf("Creating crypter: %v", err)
	}

	// Create the backup registry.
	randSrc := rand.New(rand.NewSource(time.Now().UnixNano()))
	registry, err := backup.NewRegistry(db, cfg.SdbDomain, crypter, randSrc)
	if err != nil {
		log.Fatalf("Creating registry: %v", err)
	}

	// Create the kv store.
	kvStore, err := s3_kv.NewS3KvStore(bucket)
	if err != nil {
		log.Fatalf("Creating kv store: %v", err)
	}

	// Create the blob store.
	blobStore := blob.NewKvBasedBlobStore(kvStore)
	blobStore = blob.NewCheckingStore(blobStore)
	blobStore = blob.NewEncryptingStore(crypter, blobStore)

	// Create the file saver.
	fileSaver, err := backup.NewFileSaver(blobStore, 1<<24)
	if err != nil {
		log.Fatalf("Creating file saver: %v", err)
	}

	// Create a directory saver.
	dirSaver, err := backup.NewDirectorySaver(
		blobStore,
		fileSystem,
		fileSaver)

	if err != nil {
		log.Fatalf("Creating directory saver: %v", err)
	}

	// Choose a start time for the job.
	startTime := time.Now()

	// Run the job.
	score, err := dirSaver.Save(job.BasePath, "", job.Excludes)
	if err != nil {
		log.Fatalf("Saving: %v", err)
	}

	// Register the successful backup.
	completedJob := backup.CompletedJob{
		Id:        randUint64(randSrc),
		Name:      *g_jobName,
		StartTime: startTime,
		Score:     score,
	}

	if err = registry.RecordBackup(completedJob); err != nil {
		log.Fatalf("Recoding to registry: %v", err)
	}

	fmt.Printf("Successfully backed up. ID: %16x\n", completedJob.Id)
}
<|endoftext|>


<commit_before>package asset_test

import (
	"bytes"
	"html/template"
	"io/ioutil"
	"log"
	"strings"
	"testing"

	"github.com/blue-jay/core/asset"
)

func init() {
	// Clear log messages from output (specifically TestCSSMissing())
	log.SetOutput(ioutil.Discard)
}

// TestKeysExist ensures key exist.
func TestKeysExist(t *testing.T) {
	config := asset.Info{
		Folder: "testdata",
	}

	fm := config.Map("/")

	if _, ok := fm["JS"]; !ok {
		t.Fatal("Key missing: JS")
	}

	if _, ok := fm["CSS"]; !ok {
		t.Fatal("Key missing: CSS")
	}
}

// TestCSS ensures CSS parses correctly.
func TestCSS(t *testing.T) {
	config := asset.Info{
		Folder: "testdata",
	}

	fm := config.Map("/")

	temp, err := template.New("test").Funcs(fm).Parse(`{{CSS "/test.css" "all"}}`)
	if err != nil {
		t.Fatal(err)
	}

	buf := new(bytes.Buffer)

	err = temp.Execute(buf, nil)
	if err != nil {
		t.Fatal(err)
	}

	expected := `<link media="all" rel="stylesheet" type="text/css" href="/test.css?`
	received := buf.String()

	if strings.HasPrefix(expected, received) {
		t.Errorf("\n got: %v\nwant: %v", received, expected)
	}
}

// TestCSS ensures CSS from internet parses correctly.
func TestCSSInternet(t *testing.T) {
	config := asset.Info{
		Folder: "testdata",
	}

	fm := config.Map("/")

	temp, err := template.New("test").Funcs(fm).Parse(`{{CSS "//test.css" "all"}}`)
	if err != nil {
		t.Fatal(err)
	}

	buf := new(bytes.Buffer)

	err = temp.Execute(buf, nil)
	if err != nil {
		t.Fatal(err)
	}

	expected := `<link media="all" rel="stylesheet" type="text/css" href="//test.css" />`
	received := buf.String()

	if expected != received {
		t.Errorf("\n got: %v\nwant: %v", received, expected)
	}
}

// TestCSSMissing ensures file is missing error is thrown.
func TestCSSMissing(t *testing.T) {
	config := asset.Info{
		Folder: "testdata2",
	}

	fm := config.Map("/")

	temp, err := template.New("test").Funcs(fm).Parse(`{{CSS "test.css" "all"}}`)
	if err != nil {
		t.Fatal(err)
	}

	buf := new(bytes.Buffer)

	err = temp.Execute(buf, nil)
	if err != nil {
		t.Fatal(err)
	}

	expected := `<!-- CSS Error: test.css -->`
	received := buf.String()

	if expected != received {
		t.Errorf("\n got: %v\nwant: %v", received, expected)
	}
}

// TestJS ensures JS parses correctly.
func TestJS(t *testing.T) {
	config := asset.Info{
		Folder: "testdata",
	}

	fm := config.Map("/")

	temp, err := template.New("test").Funcs(fm).Parse(`{{JS "test.js"}}`)
	if err != nil {
		t.Fatal(err)
	}

	buf := new(bytes.Buffer)

	err = temp.Execute(buf, nil)
	if err != nil {
		t.Fatal(err)
	}

	expected := `<script type="text/javascript" src="/test.js?`
	received := buf.String()

	if strings.HasPrefix(expected, received) {
		t.Errorf("\n got: %v\nwant: %v", received, expected)
	}
}

// TestJS ensures JS from internet parses correctly.
func TestJSInternet(t *testing.T) {
	config := asset.Info{
		Folder: "testdata",
	}

	fm := config.Map("/")

	temp, err := template.New("test").Funcs(fm).Parse(`{{JS "//test.js"}}`)
	if err != nil {
		t.Fatal(err)
	}

	buf := new(bytes.Buffer)

	err = temp.Execute(buf, nil)
	if err != nil {
		t.Fatal(err)
	}

	expected := `<script type="text/javascript" src="//test.js"></script>`
	received := buf.String()

	if expected != received {
		t.Errorf("\n got: %v\nwant: %v", received, expected)
	}
}

// TestJSMissing ensures file is missing error is thrown.
func TestJSMissing(t *testing.T) {
	config := asset.Info{
		Folder: "testdata2",
	}

	fm := config.Map("/")

	temp, err := template.New("test").Funcs(fm).Parse(`{{JS "test2.js"}}`)
	if err != nil {
		t.Fatal(err)
	}

	buf := new(bytes.Buffer)

	err = temp.Execute(buf, nil)
	if err != nil {
		t.Fatal(err)
	}

	expected := `<!-- JS Error: test2.js -->`
	received := buf.String()

	if expected != received {
		t.Errorf("\n got: %v\nwant: %v", received, expected)
	}
}

// TTestBaseURI ensure URI is handled correctly.
func TestBaseURI(t *testing.T) {
	config := asset.Info{
		Folder: "testdata",
	}

	fm := config.Map("/newbase/")

	temp, err := template.New("test").Funcs(fm).Parse(`{{CSS "/test.css" "all"}}`)
	if err != nil {
		t.Fatal(err)
	}

	buf := new(bytes.Buffer)

	err = temp.Execute(buf, nil)
	if err != nil {
		t.Fatal(err)
	}

	expected := `<link media="all" rel="stylesheet" type="text/css" href="/newbase/test.css?`
	received := buf.String()

	if strings.HasPrefix(expected, received) {
		t.Errorf("\n got: %v\nwant: %v", received, expected)
	}
}
<commit_msg>Fix tests in asset package<commit_after>package asset_test

import (
	"bytes"
	"html/template"
	"io/ioutil"
	"log"
	"strings"
	"testing"

	"github.com/blue-jay/core/asset"
)

func init() {
	// Clear log messages from output (specifically TestCSSMissing())
	log.SetOutput(ioutil.Discard)
}

// TestKeysExist ensures keys exist.
func TestKeysExist(t *testing.T) {
	config := asset.Info{
		Folder: "testdata",
	}

	fm := config.Map("/")

	if _, ok := fm["JS"]; !ok {
		t.Fatal("Key missing: JS")
	}

	if _, ok := fm["CSS"]; !ok {
		t.Fatal("Key missing: CSS")
	}
}

// TestCSS ensures CSS parses correctly.
func TestCSS(t *testing.T) {
	config := asset.Info{
		Folder: "testdata",
	}

	fm := config.Map("/")

	temp, err := template.New("test").Funcs(fm).Parse(`{{CSS "/test.css" "all"}}`)
	if err != nil {
		t.Fatal(err)
	}

	buf := new(bytes.Buffer)

	err = temp.Execute(buf, nil)
	if err != nil {
		t.Fatal(err)
	}

	expected := `<link media="all" rel="stylesheet" type="text/css" href="/test.css?`
	received := buf.String()

	if !strings.HasPrefix(received, expected) {
		t.Errorf("\n got: %v\nwant: %v", received, expected)
	}
}

// TestCSS ensures CSS from internet parses correctly.
func TestCSSInternet(t *testing.T) {
	config := asset.Info{
		Folder: "testdata",
	}

	fm := config.Map("/")

	temp, err := template.New("test").Funcs(fm).Parse(`{{CSS "//test.css" "all"}}`)
	if err != nil {
		t.Fatal(err)
	}

	buf := new(bytes.Buffer)

	err = temp.Execute(buf, nil)
	if err != nil {
		t.Fatal(err)
	}

	expected := `<link media="all" rel="stylesheet" type="text/css" href="//test.css" />`
	received := buf.String()

	if expected != received {
		t.Errorf("\n got: %v\nwant: %v", received, expected)
	}
}

// TestCSSMissing ensures file is missing error is thrown.
func TestCSSMissing(t *testing.T) {
	config := asset.Info{
		Folder: "testdata2",
	}

	fm := config.Map("/")

	temp, err := template.New("test").Funcs(fm).Parse(`{{CSS "test.css" "all"}}`)
	if err != nil {
		t.Fatal(err)
	}

	buf := new(bytes.Buffer)

	err = temp.Execute(buf, nil)
	if err != nil {
		t.Fatal(err)
	}

	expected := `<!-- CSS Error: test.css -->`
	received := buf.String()

	if expected != received {
		t.Errorf("\n got: %v\nwant: %v", received, expected)
	}
}

// TestJS ensures JS parses correctly.
func TestJS(t *testing.T) {
	config := asset.Info{
		Folder: "testdata",
	}

	fm := config.Map("/")

	temp, err := template.New("test").Funcs(fm).Parse(`{{JS "test.js"}}`)
	if err != nil {
		t.Fatal(err)
	}

	buf := new(bytes.Buffer)

	err = temp.Execute(buf, nil)
	if err != nil {
		t.Fatal(err)
	}

	expected := `<script type="text/javascript" src="/test.js?`
	received := buf.String()

	if !strings.HasPrefix(received, expected) {
		t.Errorf("\n got: %v\nwant: %v", received, expected)
	}
}

// TestJS ensures JS from internet parses correctly.
func TestJSInternet(t *testing.T) {
	config := asset.Info{
		Folder: "testdata",
	}

	fm := config.Map("/")

	temp, err := template.New("test").Funcs(fm).Parse(`{{JS "//test.js"}}`)
	if err != nil {
		t.Fatal(err)
	}

	buf := new(bytes.Buffer)

	err = temp.Execute(buf, nil)
	if err != nil {
		t.Fatal(err)
	}

	expected := `<script type="text/javascript" src="//test.js"></script>`
	received := buf.String()

	if expected != received {
		t.Errorf("\n got: %v\nwant: %v", received, expected)
	}
}

// TestJSMissing ensures file is missing error is thrown.
func TestJSMissing(t *testing.T) {
	config := asset.Info{
		Folder: "testdata2",
	}

	fm := config.Map("/")

	temp, err := template.New("test").Funcs(fm).Parse(`{{JS "test2.js"}}`)
	if err != nil {
		t.Fatal(err)
	}

	buf := new(bytes.Buffer)

	err = temp.Execute(buf, nil)
	if err != nil {
		t.Fatal(err)
	}

	expected := `<!-- JS Error: test2.js -->`
	received := buf.String()

	if expected != received {
		t.Errorf("\n got: %v\nwant: %v", received, expected)
	}
}

// TTestBaseURI ensure URI is handled correctly.
func TestBaseURI(t *testing.T) {
	config := asset.Info{
		Folder: "testdata",
	}

	fm := config.Map("/newbase/")

	temp, err := template.New("test").Funcs(fm).Parse(`{{CSS "/test.css" "all"}}`)
	if err != nil {
		t.Fatal(err)
	}

	buf := new(bytes.Buffer)

	err = temp.Execute(buf, nil)
	if err != nil {
		t.Fatal(err)
	}

	expected := `<link media="all" rel="stylesheet" type="text/css" href="/newbase/test.css?`
	received := buf.String()

	if !strings.HasPrefix(received, expected) {
		t.Errorf("\n got: %v\nwant: %v", received, expected)
	}
}
<|endoftext|>


<commit_before>package aws

import (
	"crypto/sha1"
	"encoding/hex"
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/iam"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsIAMServerCertificate() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsIAMServerCertificateCreate,
		Read:   resourceAwsIAMServerCertificateRead,
		Delete: resourceAwsIAMServerCertificateDelete,
		Importer: &schema.ResourceImporter{
			State: resourceAwsIAMServerCertificateImport,
		},

		Schema: map[string]*schema.Schema{
			"certificate_body": {
				Type:      schema.TypeString,
				Required:  true,
				ForceNew:  true,
				StateFunc: normalizeCert,
			},

			"certificate_chain": {
				Type:      schema.TypeString,
				Optional:  true,
				ForceNew:  true,
				StateFunc: normalizeCert,
			},

			"path": {
				Type:     schema.TypeString,
				Optional: true,
				Default:  "/",
				ForceNew: true,
			},

			"private_key": {
				Type:      schema.TypeString,
				Required:  true,
				ForceNew:  true,
				StateFunc: normalizeCert,
				Sensitive: true,
			},

			"name": {
				Type:          schema.TypeString,
				Optional:      true,
				Computed:      true,
				ForceNew:      true,
				ConflictsWith: []string{"name_prefix"},
				ValidateFunc: func(v interface{}, k string) (ws []string, errors []error) {
					value := v.(string)
					if len(value) > 128 {
						errors = append(errors, fmt.Errorf(
							"%q cannot be longer than 128 characters", k))
					}
					return
				},
			},

			"name_prefix": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				ValidateFunc: func(v interface{}, k string) (ws []string, errors []error) {
					value := v.(string)
					if len(value) > 30 {
						errors = append(errors, fmt.Errorf(
							"%q cannot be longer than 30 characters, name is limited to 128", k))
					}
					return
				},
			},

			"arn": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},
		},
	}
}

func resourceAwsIAMServerCertificateCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).iamconn

	var sslCertName string
	if v, ok := d.GetOk("name"); ok {
		sslCertName = v.(string)
	} else if v, ok := d.GetOk("name_prefix"); ok {
		sslCertName = resource.PrefixedUniqueId(v.(string))
	} else {
		sslCertName = resource.UniqueId()
	}

	createOpts := &iam.UploadServerCertificateInput{
		CertificateBody:       aws.String(d.Get("certificate_body").(string)),
		PrivateKey:            aws.String(d.Get("private_key").(string)),
		ServerCertificateName: aws.String(sslCertName),
	}

	if v, ok := d.GetOk("certificate_chain"); ok {
		createOpts.CertificateChain = aws.String(v.(string))
	}

	if v, ok := d.GetOk("path"); ok {
		createOpts.Path = aws.String(v.(string))
	}

	log.Printf("[DEBUG] Creating IAM Server Certificate with opts: %s", createOpts)
	resp, err := conn.UploadServerCertificate(createOpts)
	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			return fmt.Errorf("[WARN] Error uploading server certificate, error: %s: %s", awsErr.Code(), awsErr.Message())
		}
		return fmt.Errorf("[WARN] Error uploading server certificate, error: %s", err)
	}

	d.SetId(*resp.ServerCertificateMetadata.ServerCertificateId)
	d.Set("name", sslCertName)

	return resourceAwsIAMServerCertificateRead(d, meta)
}

func resourceAwsIAMServerCertificateRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).iamconn
	resp, err := conn.GetServerCertificate(&iam.GetServerCertificateInput{
		ServerCertificateName: aws.String(d.Get("name").(string)),
	})

	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "NoSuchEntity" {
				log.Printf("[WARN] IAM Server Cert (%s) not found, removing from state", d.Id())
				d.SetId("")
				return nil
			}
			return fmt.Errorf("[WARN] Error reading IAM Server Certificate: %s: %s", awsErr.Code(), awsErr.Message())
		}
		return fmt.Errorf("[WARN] Error reading IAM Server Certificate: %s", err)
	}

	d.SetId(*resp.ServerCertificate.ServerCertificateMetadata.ServerCertificateId)

	// these values should always be present, and have a default if not set in
	// configuration, and so safe to reference with nil checks
	d.Set("certificate_body", normalizeCert(resp.ServerCertificate.CertificateBody))

	c := normalizeCert(resp.ServerCertificate.CertificateChain)
	if c != "" {
		d.Set("certificate_chain", c)
	}

	d.Set("path", resp.ServerCertificate.ServerCertificateMetadata.Path)
	d.Set("arn", resp.ServerCertificate.ServerCertificateMetadata.Arn)

	return nil
}

func resourceAwsIAMServerCertificateDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).iamconn
	log.Printf("[INFO] Deleting IAM Server Certificate: %s", d.Id())
	err := resource.Retry(5*time.Minute, func() *resource.RetryError {
		_, err := conn.DeleteServerCertificate(&iam.DeleteServerCertificateInput{
			ServerCertificateName: aws.String(d.Get("name").(string)),
		})

		if err != nil {
			if awsErr, ok := err.(awserr.Error); ok {
				if awsErr.Code() == "DeleteConflict" && strings.Contains(awsErr.Message(), "currently in use by arn") {
					log.Printf("[WARN] Conflict deleting server certificate: %s, retrying", awsErr.Message())
					return resource.RetryableError(err)
				}
				if awsErr.Code() == "NoSuchEntity" {
					log.Printf("[WARN] IAM Server Certificate (%s) not found, removing from state", d.Id())
					d.SetId("")
					return nil
				}
			}
			return resource.NonRetryableError(err)
		}
		return nil
	})

	if err != nil {
		return err
	}

	d.SetId("")
	return nil
}

func resourceAwsIAMServerCertificateImport(
	d *schema.ResourceData, meta interface{}) ([]*schema.ResourceData, error) {
	d.Set("name", d.Id())
	// private_key can't be fetched from any API call
	return []*schema.ResourceData{d}, nil
}

func normalizeCert(cert interface{}) string {
	if cert == nil || cert == (*string)(nil) {
		return ""
	}

	var rawCert string
	switch cert.(type) {
	case string:
		rawCert = cert.(string)
	case *string:
		rawCert = *cert.(*string)
	default:
		return ""
	}

	cleanVal := sha1.Sum(stripCR([]byte(strings.TrimSpace(rawCert))))
	return hex.EncodeToString(cleanVal[:])
}

// strip CRs from raw literals. Lifted from go/scanner/scanner.go
// See https://github.com/golang/go/blob/release-branch.go1.6/src/go/scanner/scanner.go#L479
func stripCR(b []byte) []byte {
	c := make([]byte, len(b))
	i := 0
	for _, ch := range b {
		if ch != '\r' {
			c[i] = ch
			i++
		}
	}
	return c[:i]
}
<commit_msg>provider/aws: Increase timeout for retrying deletion IAM server cert (#14655)<commit_after>package aws

import (
	"crypto/sha1"
	"encoding/hex"
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/iam"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsIAMServerCertificate() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsIAMServerCertificateCreate,
		Read:   resourceAwsIAMServerCertificateRead,
		Delete: resourceAwsIAMServerCertificateDelete,
		Importer: &schema.ResourceImporter{
			State: resourceAwsIAMServerCertificateImport,
		},

		Schema: map[string]*schema.Schema{
			"certificate_body": {
				Type:      schema.TypeString,
				Required:  true,
				ForceNew:  true,
				StateFunc: normalizeCert,
			},

			"certificate_chain": {
				Type:      schema.TypeString,
				Optional:  true,
				ForceNew:  true,
				StateFunc: normalizeCert,
			},

			"path": {
				Type:     schema.TypeString,
				Optional: true,
				Default:  "/",
				ForceNew: true,
			},

			"private_key": {
				Type:      schema.TypeString,
				Required:  true,
				ForceNew:  true,
				StateFunc: normalizeCert,
				Sensitive: true,
			},

			"name": {
				Type:          schema.TypeString,
				Optional:      true,
				Computed:      true,
				ForceNew:      true,
				ConflictsWith: []string{"name_prefix"},
				ValidateFunc: func(v interface{}, k string) (ws []string, errors []error) {
					value := v.(string)
					if len(value) > 128 {
						errors = append(errors, fmt.Errorf(
							"%q cannot be longer than 128 characters", k))
					}
					return
				},
			},

			"name_prefix": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				ValidateFunc: func(v interface{}, k string) (ws []string, errors []error) {
					value := v.(string)
					if len(value) > 30 {
						errors = append(errors, fmt.Errorf(
							"%q cannot be longer than 30 characters, name is limited to 128", k))
					}
					return
				},
			},

			"arn": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},
		},
	}
}

func resourceAwsIAMServerCertificateCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).iamconn

	var sslCertName string
	if v, ok := d.GetOk("name"); ok {
		sslCertName = v.(string)
	} else if v, ok := d.GetOk("name_prefix"); ok {
		sslCertName = resource.PrefixedUniqueId(v.(string))
	} else {
		sslCertName = resource.UniqueId()
	}

	createOpts := &iam.UploadServerCertificateInput{
		CertificateBody:       aws.String(d.Get("certificate_body").(string)),
		PrivateKey:            aws.String(d.Get("private_key").(string)),
		ServerCertificateName: aws.String(sslCertName),
	}

	if v, ok := d.GetOk("certificate_chain"); ok {
		createOpts.CertificateChain = aws.String(v.(string))
	}

	if v, ok := d.GetOk("path"); ok {
		createOpts.Path = aws.String(v.(string))
	}

	log.Printf("[DEBUG] Creating IAM Server Certificate with opts: %s", createOpts)
	resp, err := conn.UploadServerCertificate(createOpts)
	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			return fmt.Errorf("[WARN] Error uploading server certificate, error: %s: %s", awsErr.Code(), awsErr.Message())
		}
		return fmt.Errorf("[WARN] Error uploading server certificate, error: %s", err)
	}

	d.SetId(*resp.ServerCertificateMetadata.ServerCertificateId)
	d.Set("name", sslCertName)

	return resourceAwsIAMServerCertificateRead(d, meta)
}

func resourceAwsIAMServerCertificateRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).iamconn
	resp, err := conn.GetServerCertificate(&iam.GetServerCertificateInput{
		ServerCertificateName: aws.String(d.Get("name").(string)),
	})

	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "NoSuchEntity" {
				log.Printf("[WARN] IAM Server Cert (%s) not found, removing from state", d.Id())
				d.SetId("")
				return nil
			}
			return fmt.Errorf("[WARN] Error reading IAM Server Certificate: %s: %s", awsErr.Code(), awsErr.Message())
		}
		return fmt.Errorf("[WARN] Error reading IAM Server Certificate: %s", err)
	}

	d.SetId(*resp.ServerCertificate.ServerCertificateMetadata.ServerCertificateId)

	// these values should always be present, and have a default if not set in
	// configuration, and so safe to reference with nil checks
	d.Set("certificate_body", normalizeCert(resp.ServerCertificate.CertificateBody))

	c := normalizeCert(resp.ServerCertificate.CertificateChain)
	if c != "" {
		d.Set("certificate_chain", c)
	}

	d.Set("path", resp.ServerCertificate.ServerCertificateMetadata.Path)
	d.Set("arn", resp.ServerCertificate.ServerCertificateMetadata.Arn)

	return nil
}

func resourceAwsIAMServerCertificateDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).iamconn
	log.Printf("[INFO] Deleting IAM Server Certificate: %s", d.Id())
	err := resource.Retry(10*time.Minute, func() *resource.RetryError {
		_, err := conn.DeleteServerCertificate(&iam.DeleteServerCertificateInput{
			ServerCertificateName: aws.String(d.Get("name").(string)),
		})

		if err != nil {
			if awsErr, ok := err.(awserr.Error); ok {
				if awsErr.Code() == "DeleteConflict" && strings.Contains(awsErr.Message(), "currently in use by arn") {
					log.Printf("[WARN] Conflict deleting server certificate: %s, retrying", awsErr.Message())
					return resource.RetryableError(err)
				}
				if awsErr.Code() == "NoSuchEntity" {
					log.Printf("[WARN] IAM Server Certificate (%s) not found, removing from state", d.Id())
					d.SetId("")
					return nil
				}
			}
			return resource.NonRetryableError(err)
		}
		return nil
	})

	if err != nil {
		return err
	}

	d.SetId("")
	return nil
}

func resourceAwsIAMServerCertificateImport(
	d *schema.ResourceData, meta interface{}) ([]*schema.ResourceData, error) {
	d.Set("name", d.Id())
	// private_key can't be fetched from any API call
	return []*schema.ResourceData{d}, nil
}

func normalizeCert(cert interface{}) string {
	if cert == nil || cert == (*string)(nil) {
		return ""
	}

	var rawCert string
	switch cert.(type) {
	case string:
		rawCert = cert.(string)
	case *string:
		rawCert = *cert.(*string)
	default:
		return ""
	}

	cleanVal := sha1.Sum(stripCR([]byte(strings.TrimSpace(rawCert))))
	return hex.EncodeToString(cleanVal[:])
}

// strip CRs from raw literals. Lifted from go/scanner/scanner.go
// See https://github.com/golang/go/blob/release-branch.go1.6/src/go/scanner/scanner.go#L479
func stripCR(b []byte) []byte {
	c := make([]byte, len(b))
	i := 0
	for _, ch := range b {
		if ch != '\r' {
			c[i] = ch
			i++
		}
	}
	return c[:i]
}
<|endoftext|>


<commit_before>// Package rollinghash/adler32 implements a rolling version of hash/adler32

package adler32

import (
	"errors"
	"github.com/chmduquesne/rollinghash"
)

const (
	mod = 65521
)

// The size of an Adler-32 checksum b bytes.
const Size = 4

// digest represents the partial evaluation of a checksum.
type digest struct {
	// invariant: (a < mod && b < mod) || a <= b
	// invariant: a + b + 255 <= 0xffffffff
	a, b uint32

	// window is treated like a circular buffer, where the oldest element
	// is indicated by d.oldest
	window []byte
	oldest int
}

// Reset resets the Hash to its initial state.
func (d *digest) Reset() { d.a, d.b = 1, 0 }

// New returns a new hash.Hash32 computing the rolling Adler-32 checksum.
// The window is copied from the last Write(). This window is only used to
// determine which is the oldest element (leaving the window). The calls
// to Roll() do not recompute the whole checksum.
func New() rollinghash.Hash32 {
	d := new(digest)
	d.Reset()
	return d
}

// Size returns the number of bytes Sum will return.
func (d *digest) Size() int { return Size }

// BlockSize returns the hash's underlying block size.
// The Write method must be able to accept any amount
// of data, but it may operate more efficiently if all
// writes are a multiple of the block size.
func (d *digest) BlockSize() int { return 1 }

// Write (via the embedded io.Writer interface) adds more data to the
// running hash. It never returns an error.
func (d *digest) Write(p []byte) (int, error) {
	// Copy the window
	d.window = make([]byte, len(p))
	copy(d.window, p)
	for _, c := range p {
		d.a += uint32(c)
		d.b += d.a
		// invariant: a <= b
		if d.b > (0xffffffff-255)/2 {
			d.a %= mod
			d.b %= mod
			// invariant: a < mod && b < mod
		} else {
			// invariant: a + b + 255 <= 2 * b + 255 <= 0xffffffff
		}
	}
	return len(d.window), nil
}

func (d *digest) Sum32() uint32 {
	if d.b >= mod {
		d.a %= mod
		d.b %= mod
	}
	return d.b<<16 | d.a
}

func (d *digest) Sum(b []byte) []byte {
	s := d.Sum32()
	b = append(b, byte(s>>24))
	b = append(b, byte(s>>16))
	b = append(b, byte(s>>8))
	b = append(b, byte(s))
	return b
}

// Roll updates the checksum of the window from the leaving byte and the
// entering byte
// See http://www.samba.org/~tridge/phd_thesis.pdf (p. 55)
// See https://groups.google.com/forum/?fromgroups=#!topic/golang-nuts/ZiBcYH3Qw1g
// See https://github.com/josvazg/slicesync/blob/master/rollingadler32.go
func (d *digest) Roll(b byte) error {
	if len(d.window) == 0 {
		return errors.New(
			"The window must be initialized with Write() first.")
	}
	// get the values for the computation and update the window
	newest := uint32(b)
	oldest := uint32(d.window[d.oldest])
	d.window[d.oldest] = b
	n := len(d.window)
	d.oldest = (d.oldest + 1) % n

	d.a += newest - oldest
	d.b += d.a - (uint32(n) * oldest) - 1
	// invariant: a <= b
	if d.b > (0xffffffff-255)/2 {
		d.a %= mod
		d.b %= mod
		// invariant: a < mod && b < mod
	} else {
		// invariant: a + b + 255 <= 2 * b + 255 <= 0xffffffff
	}
	return nil
}
<commit_msg>compacted the code<commit_after>// Package rollinghash/adler32 implements a rolling version of hash/adler32

package adler32

import (
	"errors"
	"github.com/chmduquesne/rollinghash"
)

const (
	mod = 65521
)

// The size of an Adler-32 checksum.
const Size = 4

// digest represents the partial evaluation of a checksum.
type digest struct {
	// invariant: (a < mod && b < mod) || a <= b
	// invariant: a + b + 255 <= 0xffffffff
	a, b uint32

	// window is treated like a circular buffer, where the oldest element
	// is indicated by d.oldest
	window []byte
	oldest int
}

// Reset resets the Hash to its initial state.
func (d *digest) Reset() { d.a, d.b = 1, 0 }

// New returns a new hash.Hash32 computing the rolling Adler-32 checksum.
// The window is copied from the last Write(). This window is only used to
// determine which is the oldest element (leaving the window). The calls
// to Roll() do not recompute the whole checksum.
func New() rollinghash.Hash32 {
	d := new(digest)
	d.Reset()
	return d
}

// Size returns the number of bytes Sum will return.
func (d *digest) Size() int { return Size }

// BlockSize returns the hash's underlying block size.
// The Write method must be able to accept any amount
// of data, but it may operate more efficiently if all
// writes are a multiple of the block size.
func (d *digest) BlockSize() int { return 1 }

// Write (via the embedded io.Writer interface) adds more data to the
// running hash. It never returns an error.
func (d *digest) Write(p []byte) (int, error) {
	// Copy the window
	d.window = make([]byte, len(p))
	copy(d.window, p)
	for _, c := range p {
		d.a += uint32(c)
		d.b += d.a
		// invariant: a <= b
		if d.b > (0xffffffff-255)/2 {
			d.a %= mod
			d.b %= mod
			// invariant: a < mod && b < mod
		} else {
			// invariant: a + b + 255 <= 2 * b + 255 <= 0xffffffff
		}
	}
	return len(d.window), nil
}

func (d *digest) Sum32() uint32 {
	if d.b >= mod {
		d.a %= mod
		d.b %= mod
	}
	return d.b<<16 | d.a
}

func (d *digest) Sum(b []byte) []byte {
	v := d.Sum32()
	return append(b, byte(v>>24), byte(v>>16), byte(v>>8), byte(v))
}

// Roll updates the checksum of the window from the leaving byte and the
// entering byte
// See http://www.samba.org/~tridge/phd_thesis.pdf (p. 55)
// See https://groups.google.com/forum/?fromgroups=#!topic/golang-nuts/ZiBcYH3Qw1g
// See https://github.com/josvazg/slicesync/blob/master/rollingadler32.go
func (d *digest) Roll(b byte) error {
	if len(d.window) == 0 {
		return errors.New(
			"The window must be initialized with Write() first.")
	}
	// get the values for the computation and update the window
	newest := uint32(b)
	oldest := uint32(d.window[d.oldest])
	d.window[d.oldest] = b
	n := len(d.window)
	d.oldest = (d.oldest + 1) % n

	d.a += newest - oldest
	d.b += d.a - (uint32(n) * oldest) - 1
	// invariant: a <= b
	if d.b > (0xffffffff-255)/2 {
		d.a %= mod
		d.b %= mod
		// invariant: a < mod && b < mod
	} else {
		// invariant: a + b + 255 <= 2 * b + 255 <= 0xffffffff
	}
	return nil
}
<|endoftext|>