<commit_before>package google

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccComputeProjectMetadataItem_basic(t *testing.T) {
	// Key must be unique to avoid concurrent tests interfering with each other
	key := "myKey" + acctest.RandString(10)

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckProjectMetadataItemDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccProjectMetadataItem_basic(key, "myValue"),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckProjectMetadataItem_hasMetadata(key, "myValue"),
				),
			},
		},
	})
}

func TestAccComputeProjectMetadataItem_basicWithEmptyVal(t *testing.T) {
	// Key must be unique to avoid concurrent tests interfering with each other
	key := "myKey" + acctest.RandString(10)

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckProjectMetadataItemDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccProjectMetadataItem_basic(key, ""),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckProjectMetadataItem_hasMetadata(key, ""),
				),
			},
		},
	})
}

func TestAccComputeProjectMetadataItem_basicUpdate(t *testing.T) {
	// Key must be unique to avoid concurrent tests interfering with each other
	key := "myKey" + acctest.RandString(10)

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckProjectMetadataItemDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccProjectMetadataItem_basic(key, "myValue"),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckProjectMetadataItem_hasMetadata(key, "myValue"),
				),
			},
			{
				Config: testAccProjectMetadataItem_basic(key, "myUpdatedValue"),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckProjectMetadataItem_hasMetadata(key, "myUpdatedValue"),
				),
			},
		},
	})
}

func testAccCheckProjectMetadataItem_hasMetadata(key, value string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		config := testAccProvider.Meta().(*Config)

		project, err := config.clientCompute.Projects.Get(config.Project).Do()
		if err != nil {
			return err
		}

		metadata := flattenComputeMetadata(project.CommonInstanceMetadata.Items)

		val, ok := metadata[key]
		if !ok {
			return fmt.Errorf("Unable to find a value for key '%s'", key)
		}
		if val != value {
			return fmt.Errorf("Value for key '%s' does not match. Expected '%s' but found '%s'", key, value, val)
		}
		return nil
	}
}

func testAccCheckProjectMetadataItemDestroy(s *terraform.State) error {
	config := testAccProvider.Meta().(*Config)

	project, err := config.clientCompute.Projects.Get(config.Project).Do()
	if err != nil {
		return err
	}

	metadata := flattenComputeMetadata(project.CommonInstanceMetadata.Items)

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "google_compute_project_metadata_item" {
			continue
		}

		_, ok := metadata[rs.Primary.ID]
		if ok {
			return fmt.Errorf("Metadata key/value '%s': '%s' still exist", rs.Primary.Attributes["key"], rs.Primary.Attributes["value"])
		}
	}

	return nil
}

func testAccProjectMetadataItem_basic(key, val string) string {
	return fmt.Sprintf(`
resource "google_compute_project_metadata_item" "foobar" {
  key   = "%s"
  value = "%s"
}
`, key, val)
}
<commit_msg>Add an additional test with multiple project metadata items (#222)<commit_after>package google

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccComputeProjectMetadataItem_basic(t *testing.T) {
	// Key must be unique to avoid concurrent tests interfering with each other
	key := "myKey" + acctest.RandString(10)

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckProjectMetadataItemDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccProjectMetadataItem_basic(key, "myValue"),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckProjectMetadataItem_hasMetadata(key, "myValue"),
				),
			},
		},
	})
}

func TestAccComputeProjectMetadataItem_basicMultiple(t *testing.T) {
	// Generate a config of two config keys
	config := testAccProjectMetadataItem_basic("myKey", "myValue") +
		testAccProjectMetadataItem_basic("myOtherKey", "myOtherValue")
	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckProjectMetadataItemDestroy,
		Steps: []resource.TestStep{
			{
				Config: config,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckProjectMetadataItem_hasMetadata("myKey", "myValue"),
					testAccCheckProjectMetadataItem_hasMetadata("myOtherKey", "myOtherValue"),
				),
			},
		},
	})
}

func TestAccComputeProjectMetadataItem_basicWithEmptyVal(t *testing.T) {
	// Key must be unique to avoid concurrent tests interfering with each other
	key := "myKey" + acctest.RandString(10)

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckProjectMetadataItemDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccProjectMetadataItem_basic(key, ""),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckProjectMetadataItem_hasMetadata(key, ""),
				),
			},
		},
	})
}

func TestAccComputeProjectMetadataItem_basicUpdate(t *testing.T) {
	// Key must be unique to avoid concurrent tests interfering with each other
	key := "myKey" + acctest.RandString(10)

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckProjectMetadataItemDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccProjectMetadataItem_basic(key, "myValue"),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckProjectMetadataItem_hasMetadata(key, "myValue"),
				),
			},
			{
				Config: testAccProjectMetadataItem_basic(key, "myUpdatedValue"),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckProjectMetadataItem_hasMetadata(key, "myUpdatedValue"),
				),
			},
		},
	})
}

func testAccCheckProjectMetadataItem_hasMetadata(key, value string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		config := testAccProvider.Meta().(*Config)

		project, err := config.clientCompute.Projects.Get(config.Project).Do()
		if err != nil {
			return err
		}

		metadata := flattenComputeMetadata(project.CommonInstanceMetadata.Items)

		val, ok := metadata[key]
		if !ok {
			return fmt.Errorf("Unable to find a value for key '%s'", key)
		}
		if val != value {
			return fmt.Errorf("Value for key '%s' does not match. Expected '%s' but found '%s'", key, value, val)
		}
		return nil
	}
}

func testAccCheckProjectMetadataItemDestroy(s *terraform.State) error {
	config := testAccProvider.Meta().(*Config)

	project, err := config.clientCompute.Projects.Get(config.Project).Do()
	if err != nil {
		return err
	}

	metadata := flattenComputeMetadata(project.CommonInstanceMetadata.Items)

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "google_compute_project_metadata_item" {
			continue
		}

		_, ok := metadata[rs.Primary.ID]
		if ok {
			return fmt.Errorf("Metadata key/value '%s': '%s' still exist", rs.Primary.Attributes["key"], rs.Primary.Attributes["value"])
		}
	}

	return nil
}

func testAccProjectMetadataItem_basic(key, val string) string {
	return fmt.Sprintf(`
resource "google_compute_project_metadata_item" "foobar-%s" {
  key   = "%s"
  value = "%s"
}
`, acctest.RandString(10), key, val)
}
<|endoftext|>


<commit_before>package auth

import (
	"context"
	"net/http"

	authAudit "github.com/skygeario/skygear-server/pkg/auth/dependency/audit"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/authnsession"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/forgotpwdemail"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/hook"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/mfa"
	mfaPQ "github.com/skygeario/skygear-server/pkg/auth/dependency/mfa/pq"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/passwordhistory"
	pqPWHistory "github.com/skygeario/skygear-server/pkg/auth/dependency/passwordhistory/pq"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/principal"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/principal/customtoken"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/principal/oauth"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/principal/password"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/sso"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/urlprefix"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/userprofile"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/userverify"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/welcemail"
	authTemplate "github.com/skygeario/skygear-server/pkg/auth/template"
	"github.com/skygeario/skygear-server/pkg/core/apiclientconfig"
	"github.com/skygeario/skygear-server/pkg/core/async"
	"github.com/skygeario/skygear-server/pkg/core/audit"
	coreAuth "github.com/skygeario/skygear-server/pkg/core/auth"
	"github.com/skygeario/skygear-server/pkg/core/auth/authinfo"
	pqAuthInfo "github.com/skygeario/skygear-server/pkg/core/auth/authinfo/pq"
	"github.com/skygeario/skygear-server/pkg/core/auth/session"
	redisSession "github.com/skygeario/skygear-server/pkg/core/auth/session/redis"
	"github.com/skygeario/skygear-server/pkg/core/config"
	"github.com/skygeario/skygear-server/pkg/core/db"
	"github.com/skygeario/skygear-server/pkg/core/handler"
	"github.com/skygeario/skygear-server/pkg/core/logging"
	"github.com/skygeario/skygear-server/pkg/core/mail"
	"github.com/skygeario/skygear-server/pkg/core/sentry"
	"github.com/skygeario/skygear-server/pkg/core/sms"
	"github.com/skygeario/skygear-server/pkg/core/template"
	"github.com/skygeario/skygear-server/pkg/core/time"
	"github.com/skygeario/skygear-server/pkg/core/validation"
)

type DependencyMap struct {
	EnableFileSystemTemplate bool
	Validator                *validation.Validator
	AssetGearLoader          *template.AssetGearLoader
	AsyncTaskExecutor        *async.Executor
	UseInsecureCookie        bool
	DefaultConfiguration     config.DefaultConfiguration
	ReservedNameChecker      *password.ReservedNameChecker
}

// Provide provides dependency instance by name
// nolint: gocyclo, golint
func (m DependencyMap) Provide(
	dependencyName string,
	request *http.Request,
	ctx context.Context,
	requestID string,
	tc config.TenantConfiguration,
) interface{} {
	// populate default
	userConfig := tc.UserConfig
	if !userConfig.SMTP.IsValid() {
		userConfig.SMTP = m.DefaultConfiguration.SMTP
	}

	if !userConfig.Twilio.IsValid() {
		userConfig.Twilio = m.DefaultConfiguration.Twilio
	}

	if !userConfig.Nexmo.IsValid() {
		userConfig.Nexmo = m.DefaultConfiguration.Nexmo
	}

	// To avoid mutating tc
	tConfig := tc
	tConfig.UserConfig = userConfig

	newLoggerFactory := func() logging.Factory {
		logHook := logging.NewDefaultLogHook(tConfig.DefaultSensitiveLoggerValues())
		sentryHook := sentry.NewLogHookFromContext(ctx)
		if request == nil {
			return logging.NewFactoryFromRequestID(requestID, logHook, sentryHook)
		} else {
			return logging.NewFactoryFromRequest(request, logHook, sentryHook)
		}
	}

	newSQLBuilder := func() db.SQLBuilder {
		return db.NewSQLBuilder("auth", tConfig.AppConfig.DatabaseSchema, tConfig.AppID)
	}

	newSQLExecutor := func() db.SQLExecutor {
		return db.NewSQLExecutor(ctx, db.NewContextWithContext(ctx, tConfig))
	}

	newTimeProvider := func() time.Provider {
		return time.NewProvider()
	}

	newAuthContext := func() coreAuth.ContextGetter {
		return coreAuth.NewContextGetterWithContext(ctx)
	}

	newPasswordStore := func() password.Store {
		return password.NewStore(
			newSQLBuilder(),
			newSQLExecutor(),
		)
	}

	newPasswordHistoryStore := func() passwordhistory.Store {
		return pqPWHistory.NewPasswordHistoryStore(
			newSQLBuilder(),
			newSQLExecutor(),
			newLoggerFactory(),
		)
	}

	newTemplateEngine := func() *template.Engine {
		return authTemplate.NewEngineWithConfig(
			tConfig,
			m.EnableFileSystemTemplate,
			m.AssetGearLoader,
		)
	}

	newAuthInfoStore := func() authinfo.Store {
		return pqAuthInfo.NewAuthInfoStore(
			db.NewSQLBuilder("core", tConfig.AppConfig.DatabaseSchema, tConfig.AppID),
			newSQLExecutor(),
		)
	}

	newUserProfileStore := func() userprofile.Store {
		return userprofile.NewUserProfileStore(
			newSQLBuilder(),
			newSQLExecutor(),
		)
	}

	// TODO:
	// from tConfig
	isPasswordHistoryEnabled := func() bool {
		return tConfig.UserConfig.PasswordPolicy.HistorySize > 0 ||
			tConfig.UserConfig.PasswordPolicy.HistoryDays > 0
	}

	newPasswordAuthProvider := func() password.Provider {
		return password.NewProvider(
			newPasswordStore(),
			newPasswordHistoryStore(),
			newLoggerFactory(),
			tConfig.UserConfig.Auth.LoginIDKeys,
			tConfig.UserConfig.Auth.LoginIDTypes,
			tConfig.UserConfig.Auth.AllowedRealms,
			isPasswordHistoryEnabled(),
			m.ReservedNameChecker,
		)
	}

	newCustomTokenAuthProvider := func() customtoken.Provider {
		return customtoken.NewProvider(
			newSQLBuilder(),
			newSQLExecutor(),
			tConfig.UserConfig.SSO.CustomToken,
		)
	}

	newOAuthAuthProvider := func() oauth.Provider {
		return oauth.NewProvider(
			newSQLBuilder(),
			newSQLExecutor(),
		)
	}

	newHookProvider := func() hook.Provider {
		return hook.NewProvider(
			requestID,
			urlprefix.NewProvider(request),
			hook.NewStore(newSQLBuilder(), newSQLExecutor()),
			newAuthContext(),
			newTimeProvider(),
			newAuthInfoStore(),
			newUserProfileStore(),
			hook.NewDeliverer(
				&tConfig,
				newTimeProvider(),
				hook.NewMutator(
					tConfig.UserConfig.UserVerification,
					newPasswordAuthProvider(),
					newAuthInfoStore(),
					newUserProfileStore(),
				),
			),
			newLoggerFactory(),
		)
	}

	newSessionProvider := func() session.Provider {
		return session.NewProvider(
			request,
			redisSession.NewStore(ctx, tConfig.AppID, newTimeProvider(), newLoggerFactory()),
			redisSession.NewEventStore(ctx, tConfig.AppID),
			newAuthContext(),
			tConfig.UserConfig.Clients,
		)
	}

	newIdentityProvider := func() principal.IdentityProvider {
		return principal.NewIdentityProvider(
			newSQLBuilder(),
			newSQLExecutor(),
			newCustomTokenAuthProvider(),
			newOAuthAuthProvider(),
			newPasswordAuthProvider(),
		)
	}

	newSessionWriter := func() session.Writer {
		return session.NewWriter(
			newAuthContext(),
			tConfig.UserConfig.Clients,
			tConfig.UserConfig.MFA,
			m.UseInsecureCookie,
		)
	}

	newSMSClient := func() sms.Client {
		return sms.NewClient(tConfig.UserConfig)
	}

	newMailSender := func() mail.Sender {
		return mail.NewSender(tConfig.UserConfig.SMTP)
	}

	newMFAProvider := func() mfa.Provider {
		return mfa.NewProvider(
			mfaPQ.NewStore(
				tConfig.UserConfig.MFA,
				newSQLBuilder(),
				newSQLExecutor(),
				newTimeProvider(),
			),
			tConfig.UserConfig.MFA,
			newTimeProvider(),
			mfa.NewSender(
				tConfig,
				newSMSClient(),
				newMailSender(),
				newTemplateEngine(),
			),
		)
	}

	switch dependencyName {
	case "AuthContextGetter":
		return newAuthContext()
	case "AuthContextSetter":
		return coreAuth.NewContextSetterWithContext(ctx)
	case "TxContext":
		return db.NewTxContextWithContext(ctx, tConfig)
	case "LoggerFactory":
		return newLoggerFactory()
	case "RequireAuthz":
		return handler.NewRequireAuthzFactory(newAuthContext(), newLoggerFactory())
	case "Validator":
		return m.Validator
	case "SessionProvider":
		return newSessionProvider()
	case "SessionWriter":
		return newSessionWriter()
	case "MFAProvider":
		return newMFAProvider()
	case "AuthnSessionProvider":
		return authnsession.NewProvider(
			newAuthContext(),
			tConfig.UserConfig.MFA,
			tConfig.UserConfig.Auth.AuthenticationSession,
			newTimeProvider(),
			newMFAProvider(),
			newAuthInfoStore(),
			newSessionProvider(),
			newSessionWriter(),
			newIdentityProvider(),
			newHookProvider(),
			newUserProfileStore(),
		)
	case "AuthInfoStore":
		return newAuthInfoStore()
	case "PasswordChecker":
		return &authAudit.PasswordChecker{
			PwMinLength:         tConfig.UserConfig.PasswordPolicy.MinLength,
			PwUppercaseRequired: tConfig.UserConfig.PasswordPolicy.UppercaseRequired,
			PwLowercaseRequired: tConfig.UserConfig.PasswordPolicy.LowercaseRequired,
			PwDigitRequired:     tConfig.UserConfig.PasswordPolicy.DigitRequired,
			PwSymbolRequired:    tConfig.UserConfig.PasswordPolicy.SymbolRequired,
			PwMinGuessableLevel: tConfig.UserConfig.PasswordPolicy.MinimumGuessableLevel,
			PwExcludedKeywords:  tConfig.UserConfig.PasswordPolicy.ExcludedKeywords,
			//PwExcludedFields:       tConfig.UserConfig.PasswordPolicy.ExcludedFields,
			PwHistorySize:          tConfig.UserConfig.PasswordPolicy.HistorySize,
			PwHistoryDays:          tConfig.UserConfig.PasswordPolicy.HistoryDays,
			PasswordHistoryEnabled: tConfig.UserConfig.PasswordPolicy.HistorySize > 0 || tConfig.UserConfig.PasswordPolicy.HistoryDays > 0,
			PasswordHistoryStore:   newPasswordHistoryStore(),
		}
	case "PwHousekeeper":
		return authAudit.NewPwHousekeeper(
			newPasswordHistoryStore(),
			newLoggerFactory(),
			tConfig.UserConfig.PasswordPolicy.HistorySize,
			tConfig.UserConfig.PasswordPolicy.HistoryDays,
			isPasswordHistoryEnabled(),
		)
	case "PasswordAuthProvider":
		return newPasswordAuthProvider()
	case "CustomTokenAuthProvider":
		return newCustomTokenAuthProvider()
	case "HandlerLogger":
		return newLoggerFactory().NewLogger("handler")
	case "UserProfileStore":
		return newUserProfileStore()
	case "ForgotPasswordEmailSender":
		return forgotpwdemail.NewDefaultSender(tConfig, urlprefix.NewProvider(request).Value(), newMailSender(), newTemplateEngine())
	case "ForgotPasswordCodeGenerator":
		return &forgotpwdemail.CodeGenerator{MasterKey: tConfig.UserConfig.MasterKey}
	case "ForgotPasswordSecureMatch":
		return tConfig.UserConfig.ForgotPassword.SecureMatch
	case "ResetPasswordHTMLProvider":
		return forgotpwdemail.NewResetPasswordHTMLProvider(urlprefix.NewProvider(request).Value(), tConfig.UserConfig.ForgotPassword, newTemplateEngine())
	case "WelcomeEmailEnabled":
		return tConfig.UserConfig.WelcomeEmail.Enabled
	case "WelcomeEmailDestination":
		return tConfig.UserConfig.WelcomeEmail.Destination
	case "WelcomeEmailSender":
		return welcemail.NewDefaultSender(tConfig, newMailSender(), newTemplateEngine())
	case "UserVerifyCodeSenderFactory":
		return userverify.NewDefaultUserVerifyCodeSenderFactory(
			tConfig,
			newTemplateEngine(),
			newMailSender(),
			newSMSClient(),
		)
	case "AutoSendUserVerifyCodeOnSignup":
		return tConfig.UserConfig.UserVerification.AutoSendOnSignup
	case "UserVerifyLoginIDKeys":
		return tConfig.UserConfig.UserVerification.LoginIDKeys
	case "UserVerificationProvider":
		return userverify.NewProvider(
			userverify.NewCodeGenerator(tConfig),
			userverify.NewStore(
				newSQLBuilder(),
				newSQLExecutor(),
			),
			tConfig.UserConfig.UserVerification,
			newTimeProvider(),
		)
	case "VerifyHTMLProvider":
		return userverify.NewVerifyHTMLProvider(tConfig.UserConfig.UserVerification, newTemplateEngine())
	case "AuditTrail":
		trail, err := audit.NewTrail(tConfig.UserConfig.UserAudit.Enabled, tConfig.UserConfig.UserAudit.TrailHandlerURL)
		if err != nil {
			panic(err)
		}
		return trail
	case "SSOOAuthProviderFactory":
		return sso.NewOAuthProviderFactory(tConfig, urlprefix.NewProvider(request))
	case "SSOProvider":
		return sso.NewProvider(
			tConfig.AppID,
			tConfig.UserConfig.SSO.OAuth,
		)
	case "OAuthAuthProvider":
		return newOAuthAuthProvider()
	case "IdentityProvider":
		return newIdentityProvider()
	case "AuthHandlerHTMLProvider":
		return sso.NewAuthHandlerHTMLProvider(urlprefix.NewProvider(request).Value())
	case "AsyncTaskQueue":
		return async.NewQueue(ctx, requestID, tConfig, m.AsyncTaskExecutor)
	case "HookProvider":
		return newHookProvider()
	case "CustomTokenConfiguration":
		return tConfig.UserConfig.SSO.CustomToken
	case "OAuthConfiguration":
		return tConfig.UserConfig.SSO.OAuth
	case "AuthConfiguration":
		return *tConfig.UserConfig.Auth
	case "MFAConfiguration":
		return *tConfig.UserConfig.MFA
	case "APIClientConfigurationProvider":
		return apiclientconfig.NewProvider(newAuthContext(), tConfig)
	case "URLPrefix":
		return urlprefix.NewProvider(request).Value()
	case "TemplateEngine":
		return newTemplateEngine()
	default:
		return nil
	}
}
<commit_msg>Make HookProvider a scoped dependency<commit_after>package auth

import (
	"context"
	"net/http"

	authAudit "github.com/skygeario/skygear-server/pkg/auth/dependency/audit"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/authnsession"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/forgotpwdemail"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/hook"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/mfa"
	mfaPQ "github.com/skygeario/skygear-server/pkg/auth/dependency/mfa/pq"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/passwordhistory"
	pqPWHistory "github.com/skygeario/skygear-server/pkg/auth/dependency/passwordhistory/pq"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/principal"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/principal/customtoken"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/principal/oauth"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/principal/password"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/sso"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/urlprefix"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/userprofile"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/userverify"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/welcemail"
	authTemplate "github.com/skygeario/skygear-server/pkg/auth/template"
	"github.com/skygeario/skygear-server/pkg/core/apiclientconfig"
	"github.com/skygeario/skygear-server/pkg/core/async"
	"github.com/skygeario/skygear-server/pkg/core/audit"
	coreAuth "github.com/skygeario/skygear-server/pkg/core/auth"
	"github.com/skygeario/skygear-server/pkg/core/auth/authinfo"
	pqAuthInfo "github.com/skygeario/skygear-server/pkg/core/auth/authinfo/pq"
	"github.com/skygeario/skygear-server/pkg/core/auth/session"
	redisSession "github.com/skygeario/skygear-server/pkg/core/auth/session/redis"
	"github.com/skygeario/skygear-server/pkg/core/config"
	"github.com/skygeario/skygear-server/pkg/core/db"
	"github.com/skygeario/skygear-server/pkg/core/handler"
	"github.com/skygeario/skygear-server/pkg/core/inject"
	"github.com/skygeario/skygear-server/pkg/core/logging"
	"github.com/skygeario/skygear-server/pkg/core/mail"
	"github.com/skygeario/skygear-server/pkg/core/sentry"
	"github.com/skygeario/skygear-server/pkg/core/sms"
	"github.com/skygeario/skygear-server/pkg/core/template"
	"github.com/skygeario/skygear-server/pkg/core/time"
	"github.com/skygeario/skygear-server/pkg/core/validation"
)

type DependencyMap struct {
	EnableFileSystemTemplate bool
	Validator                *validation.Validator
	AssetGearLoader          *template.AssetGearLoader
	AsyncTaskExecutor        *async.Executor
	UseInsecureCookie        bool
	DefaultConfiguration     config.DefaultConfiguration
	ReservedNameChecker      *password.ReservedNameChecker
}

// Provide provides dependency instance by name
// nolint: gocyclo, golint
func (m DependencyMap) Provide(
	dependencyName string,
	request *http.Request,
	ctx context.Context,
	requestID string,
	tc config.TenantConfiguration,
) interface{} {
	// populate default
	userConfig := tc.UserConfig
	if !userConfig.SMTP.IsValid() {
		userConfig.SMTP = m.DefaultConfiguration.SMTP
	}

	if !userConfig.Twilio.IsValid() {
		userConfig.Twilio = m.DefaultConfiguration.Twilio
	}

	if !userConfig.Nexmo.IsValid() {
		userConfig.Nexmo = m.DefaultConfiguration.Nexmo
	}

	// To avoid mutating tc
	tConfig := tc
	tConfig.UserConfig = userConfig

	newLoggerFactory := func() logging.Factory {
		logHook := logging.NewDefaultLogHook(tConfig.DefaultSensitiveLoggerValues())
		sentryHook := sentry.NewLogHookFromContext(ctx)
		if request == nil {
			return logging.NewFactoryFromRequestID(requestID, logHook, sentryHook)
		} else {
			return logging.NewFactoryFromRequest(request, logHook, sentryHook)
		}
	}

	newSQLBuilder := func() db.SQLBuilder {
		return db.NewSQLBuilder("auth", tConfig.AppConfig.DatabaseSchema, tConfig.AppID)
	}

	newSQLExecutor := func() db.SQLExecutor {
		return db.NewSQLExecutor(ctx, db.NewContextWithContext(ctx, tConfig))
	}

	newTimeProvider := func() time.Provider {
		return time.NewProvider()
	}

	newAuthContext := func() coreAuth.ContextGetter {
		return coreAuth.NewContextGetterWithContext(ctx)
	}

	newPasswordStore := func() password.Store {
		return password.NewStore(
			newSQLBuilder(),
			newSQLExecutor(),
		)
	}

	newPasswordHistoryStore := func() passwordhistory.Store {
		return pqPWHistory.NewPasswordHistoryStore(
			newSQLBuilder(),
			newSQLExecutor(),
			newLoggerFactory(),
		)
	}

	newTemplateEngine := func() *template.Engine {
		return authTemplate.NewEngineWithConfig(
			tConfig,
			m.EnableFileSystemTemplate,
			m.AssetGearLoader,
		)
	}

	newAuthInfoStore := func() authinfo.Store {
		return pqAuthInfo.NewAuthInfoStore(
			db.NewSQLBuilder("core", tConfig.AppConfig.DatabaseSchema, tConfig.AppID),
			newSQLExecutor(),
		)
	}

	newUserProfileStore := func() userprofile.Store {
		return userprofile.NewUserProfileStore(
			newSQLBuilder(),
			newSQLExecutor(),
		)
	}

	// TODO:
	// from tConfig
	isPasswordHistoryEnabled := func() bool {
		return tConfig.UserConfig.PasswordPolicy.HistorySize > 0 ||
			tConfig.UserConfig.PasswordPolicy.HistoryDays > 0
	}

	newPasswordAuthProvider := func() password.Provider {
		return password.NewProvider(
			newPasswordStore(),
			newPasswordHistoryStore(),
			newLoggerFactory(),
			tConfig.UserConfig.Auth.LoginIDKeys,
			tConfig.UserConfig.Auth.LoginIDTypes,
			tConfig.UserConfig.Auth.AllowedRealms,
			isPasswordHistoryEnabled(),
			m.ReservedNameChecker,
		)
	}

	newCustomTokenAuthProvider := func() customtoken.Provider {
		return customtoken.NewProvider(
			newSQLBuilder(),
			newSQLExecutor(),
			tConfig.UserConfig.SSO.CustomToken,
		)
	}

	newOAuthAuthProvider := func() oauth.Provider {
		return oauth.NewProvider(
			newSQLBuilder(),
			newSQLExecutor(),
		)
	}

	newHookProvider := func() hook.Provider {
		return inject.Scoped(ctx, "HookProvider", func() interface{} {
			return hook.NewProvider(
				requestID,
				urlprefix.NewProvider(request),
				hook.NewStore(newSQLBuilder(), newSQLExecutor()),
				newAuthContext(),
				newTimeProvider(),
				newAuthInfoStore(),
				newUserProfileStore(),
				hook.NewDeliverer(
					&tConfig,
					newTimeProvider(),
					hook.NewMutator(
						tConfig.UserConfig.UserVerification,
						newPasswordAuthProvider(),
						newAuthInfoStore(),
						newUserProfileStore(),
					),
				),
				newLoggerFactory(),
			)
		})().(hook.Provider)
	}

	newSessionProvider := func() session.Provider {
		return session.NewProvider(
			request,
			redisSession.NewStore(ctx, tConfig.AppID, newTimeProvider(), newLoggerFactory()),
			redisSession.NewEventStore(ctx, tConfig.AppID),
			newAuthContext(),
			tConfig.UserConfig.Clients,
		)
	}

	newIdentityProvider := func() principal.IdentityProvider {
		return principal.NewIdentityProvider(
			newSQLBuilder(),
			newSQLExecutor(),
			newCustomTokenAuthProvider(),
			newOAuthAuthProvider(),
			newPasswordAuthProvider(),
		)
	}

	newSessionWriter := func() session.Writer {
		return session.NewWriter(
			newAuthContext(),
			tConfig.UserConfig.Clients,
			tConfig.UserConfig.MFA,
			m.UseInsecureCookie,
		)
	}

	newSMSClient := func() sms.Client {
		return sms.NewClient(tConfig.UserConfig)
	}

	newMailSender := func() mail.Sender {
		return mail.NewSender(tConfig.UserConfig.SMTP)
	}

	newMFAProvider := func() mfa.Provider {
		return mfa.NewProvider(
			mfaPQ.NewStore(
				tConfig.UserConfig.MFA,
				newSQLBuilder(),
				newSQLExecutor(),
				newTimeProvider(),
			),
			tConfig.UserConfig.MFA,
			newTimeProvider(),
			mfa.NewSender(
				tConfig,
				newSMSClient(),
				newMailSender(),
				newTemplateEngine(),
			),
		)
	}

	switch dependencyName {
	case "AuthContextGetter":
		return newAuthContext()
	case "AuthContextSetter":
		return coreAuth.NewContextSetterWithContext(ctx)
	case "TxContext":
		return db.NewTxContextWithContext(ctx, tConfig)
	case "LoggerFactory":
		return newLoggerFactory()
	case "RequireAuthz":
		return handler.NewRequireAuthzFactory(newAuthContext(), newLoggerFactory())
	case "Validator":
		return m.Validator
	case "SessionProvider":
		return newSessionProvider()
	case "SessionWriter":
		return newSessionWriter()
	case "MFAProvider":
		return newMFAProvider()
	case "AuthnSessionProvider":
		return authnsession.NewProvider(
			newAuthContext(),
			tConfig.UserConfig.MFA,
			tConfig.UserConfig.Auth.AuthenticationSession,
			newTimeProvider(),
			newMFAProvider(),
			newAuthInfoStore(),
			newSessionProvider(),
			newSessionWriter(),
			newIdentityProvider(),
			newHookProvider(),
			newUserProfileStore(),
		)
	case "AuthInfoStore":
		return newAuthInfoStore()
	case "PasswordChecker":
		return &authAudit.PasswordChecker{
			PwMinLength:         tConfig.UserConfig.PasswordPolicy.MinLength,
			PwUppercaseRequired: tConfig.UserConfig.PasswordPolicy.UppercaseRequired,
			PwLowercaseRequired: tConfig.UserConfig.PasswordPolicy.LowercaseRequired,
			PwDigitRequired:     tConfig.UserConfig.PasswordPolicy.DigitRequired,
			PwSymbolRequired:    tConfig.UserConfig.PasswordPolicy.SymbolRequired,
			PwMinGuessableLevel: tConfig.UserConfig.PasswordPolicy.MinimumGuessableLevel,
			PwExcludedKeywords:  tConfig.UserConfig.PasswordPolicy.ExcludedKeywords,
			//PwExcludedFields:       tConfig.UserConfig.PasswordPolicy.ExcludedFields,
			PwHistorySize:          tConfig.UserConfig.PasswordPolicy.HistorySize,
			PwHistoryDays:          tConfig.UserConfig.PasswordPolicy.HistoryDays,
			PasswordHistoryEnabled: tConfig.UserConfig.PasswordPolicy.HistorySize > 0 || tConfig.UserConfig.PasswordPolicy.HistoryDays > 0,
			PasswordHistoryStore:   newPasswordHistoryStore(),
		}
	case "PwHousekeeper":
		return authAudit.NewPwHousekeeper(
			newPasswordHistoryStore(),
			newLoggerFactory(),
			tConfig.UserConfig.PasswordPolicy.HistorySize,
			tConfig.UserConfig.PasswordPolicy.HistoryDays,
			isPasswordHistoryEnabled(),
		)
	case "PasswordAuthProvider":
		return newPasswordAuthProvider()
	case "CustomTokenAuthProvider":
		return newCustomTokenAuthProvider()
	case "HandlerLogger":
		return newLoggerFactory().NewLogger("handler")
	case "UserProfileStore":
		return newUserProfileStore()
	case "ForgotPasswordEmailSender":
		return forgotpwdemail.NewDefaultSender(tConfig, urlprefix.NewProvider(request).Value(), newMailSender(), newTemplateEngine())
	case "ForgotPasswordCodeGenerator":
		return &forgotpwdemail.CodeGenerator{MasterKey: tConfig.UserConfig.MasterKey}
	case "ForgotPasswordSecureMatch":
		return tConfig.UserConfig.ForgotPassword.SecureMatch
	case "ResetPasswordHTMLProvider":
		return forgotpwdemail.NewResetPasswordHTMLProvider(urlprefix.NewProvider(request).Value(), tConfig.UserConfig.ForgotPassword, newTemplateEngine())
	case "WelcomeEmailEnabled":
		return tConfig.UserConfig.WelcomeEmail.Enabled
	case "WelcomeEmailDestination":
		return tConfig.UserConfig.WelcomeEmail.Destination
	case "WelcomeEmailSender":
		return welcemail.NewDefaultSender(tConfig, newMailSender(), newTemplateEngine())
	case "UserVerifyCodeSenderFactory":
		return userverify.NewDefaultUserVerifyCodeSenderFactory(
			tConfig,
			newTemplateEngine(),
			newMailSender(),
			newSMSClient(),
		)
	case "AutoSendUserVerifyCodeOnSignup":
		return tConfig.UserConfig.UserVerification.AutoSendOnSignup
	case "UserVerifyLoginIDKeys":
		return tConfig.UserConfig.UserVerification.LoginIDKeys
	case "UserVerificationProvider":
		return userverify.NewProvider(
			userverify.NewCodeGenerator(tConfig),
			userverify.NewStore(
				newSQLBuilder(),
				newSQLExecutor(),
			),
			tConfig.UserConfig.UserVerification,
			newTimeProvider(),
		)
	case "VerifyHTMLProvider":
		return userverify.NewVerifyHTMLProvider(tConfig.UserConfig.UserVerification, newTemplateEngine())
	case "AuditTrail":
		trail, err := audit.NewTrail(tConfig.UserConfig.UserAudit.Enabled, tConfig.UserConfig.UserAudit.TrailHandlerURL)
		if err != nil {
			panic(err)
		}
		return trail
	case "SSOOAuthProviderFactory":
		return sso.NewOAuthProviderFactory(tConfig, urlprefix.NewProvider(request))
	case "SSOProvider":
		return sso.NewProvider(
			tConfig.AppID,
			tConfig.UserConfig.SSO.OAuth,
		)
	case "OAuthAuthProvider":
		return newOAuthAuthProvider()
	case "IdentityProvider":
		return newIdentityProvider()
	case "AuthHandlerHTMLProvider":
		return sso.NewAuthHandlerHTMLProvider(urlprefix.NewProvider(request).Value())
	case "AsyncTaskQueue":
		return async.NewQueue(ctx, requestID, tConfig, m.AsyncTaskExecutor)
	case "HookProvider":
		return newHookProvider()
	case "CustomTokenConfiguration":
		return tConfig.UserConfig.SSO.CustomToken
	case "OAuthConfiguration":
		return tConfig.UserConfig.SSO.OAuth
	case "AuthConfiguration":
		return *tConfig.UserConfig.Auth
	case "MFAConfiguration":
		return *tConfig.UserConfig.MFA
	case "APIClientConfigurationProvider":
		return apiclientconfig.NewProvider(newAuthContext(), tConfig)
	case "URLPrefix":
		return urlprefix.NewProvider(request).Value()
	case "TemplateEngine":
		return newTemplateEngine()
	default:
		return nil
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gcb

//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate

import (
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/blang/semver"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"

	"k8s.io/release/pkg/gcp"
	"k8s.io/release/pkg/gcp/auth"
	"k8s.io/release/pkg/gcp/build"
	"k8s.io/release/pkg/git"
	"k8s.io/release/pkg/kubecross"
	"k8s.io/release/pkg/release"
	"sigs.k8s.io/release-utils/util"
)

// GCB is the main structure of this package.
type GCB struct {
	options        *Options
	repoClient     Repository
	versionClient  Version
	listJobsClient ListJobs
	releaseClient  Release
}

// New creates a new `*GCB` instance.
func New(options *Options) *GCB {
	return &GCB{
		repoClient:     release.NewRepo(),
		versionClient:  release.NewVersion(),
		listJobsClient: &defaultListJobsClient{},
		releaseClient:  &defaultReleaseClient{},
		options:        options,
	}
}

// SetRepoClient can be used to set the internal `Repository` client.
func (g *GCB) SetRepoClient(client Repository) {
	g.repoClient = client
}

// SetVersionClient can be used to set the internal `Version` client.
func (g *GCB) SetVersionClient(client Version) {
	g.versionClient = client
}

// SetListJobsClient can be used to set the internal `ListJobs` client.
func (g *GCB) SetListJobsClient(client ListJobs) {
	g.listJobsClient = client
}

// SetReleaseClient can be used to set the internal `Release` client.
func (g *GCB) SetReleaseClient(client Release) {
	g.releaseClient = client
}

type Options struct {
	build.Options
	NoMock       bool
	Stage        bool
	Release      bool
	Stream       bool
	BuildAtHead  bool
	Branch       string
	ReleaseType  string
	BuildVersion string
	GcpUser      string
	LogLevel     string
	LastJobs     int64
}

// NewDefaultOptions returns a new default `*Options` instance.
func NewDefaultOptions() *Options {
	return &Options{
		LogLevel: logrus.StandardLogger().GetLevel().String(),
		Options:  *build.NewDefaultOptions(),
	}
}

//counterfeiter:generate . Repository
type Repository interface {
	Open() error
	CheckState(string, string, string, bool) error
	GetTag() (string, error)
}

//counterfeiter:generate . Version
type Version interface {
	GetKubeVersionForBranch(release.VersionType, string) (string, error)
}

//counterfeiter:generate . ListJobs
type ListJobs interface {
	ListJobs(project string, lastJobs int64) error
}

type defaultListJobsClient struct{}

func (d *defaultListJobsClient) ListJobs(project string, lastJobs int64) error {
	return build.ListJobs(project, lastJobs)
}

//counterfeiter:generate . Release
type Release interface {
	NeedsCreation(
		branch, releaseType string, buildVersion semver.Version,
	) (createReleaseBranch bool, err error)
	GenerateReleaseVersion(
		releaseType, version, branch string, branchFromMaster bool,
	) (*release.Versions, error)
}

type defaultReleaseClient struct{}

func (*defaultReleaseClient) NeedsCreation(
	branch, releaseType string, buildVersion semver.Version,
) (createReleaseBranch bool, err error) {
	return release.NewBranchChecker().NeedsCreation(
		branch, releaseType, buildVersion,
	)
}

func (*defaultReleaseClient) GenerateReleaseVersion(
	releaseType, version, branch string, branchFromMaster bool,
) (*release.Versions, error) {
	return release.GenerateReleaseVersion(
		releaseType, version, branch, branchFromMaster,
	)
}

// Validate checks if the Options are valid.
func (o *Options) Validate() error {
	if o.Stage && o.Release {
		return errors.New("cannot specify both the 'stage' and 'release' flag; resubmit with only one build type selected")
	}

	if o.Branch == git.DefaultBranch {
		if o.ReleaseType == release.ReleaseTypeRC || o.ReleaseType == release.ReleaseTypeOfficial {
			return errors.Errorf("cannot cut a release candidate or an official release from %s", git.DefaultBranch)
		}
	} else {
		if o.ReleaseType == release.ReleaseTypeAlpha || o.ReleaseType == release.ReleaseTypeBeta {
			return errors.New("cannot cut an alpha or beta release from a release branch")
		}
	}

	if o.BuildVersion != "" && o.BuildAtHead {
		return errors.New("cannot specify both the 'build-version' and 'build-at-head' flag; resubmit with only one build option selected")
	}

	if o.BuildAtHead && o.Release {
		return errors.New("cannot specify both the 'build-at-head' flag together with the 'release' flag; resubmit with a 'build-version' flag set")
	}

	return nil
}

// Submit is the main method responsible for submitting release jobs to GCB.
func (g *GCB) Submit() error {
	if err := g.options.Validate(); err != nil {
		return errors.Wrap(err, "validating GCB options")
	}

	toolOrg := release.GetToolOrg()
	toolRepo := release.GetToolRepo()
	toolRef := release.GetToolRef()

	if err := gcp.PreCheck(); err != nil {
		return errors.Wrap(err, "pre-checking for GCP package usage")
	}

	if err := g.repoClient.Open(); err != nil {
		return errors.Wrap(err, "open release repo")
	}

	if err := g.repoClient.CheckState(toolOrg, toolRepo, toolRef, g.options.NoMock); err != nil {
		return errors.Wrap(err, "verifying repository state")
	}

	logrus.Infof("Running GCB with the following options: %+v", g.options)

	g.options.NoSource = true
	g.options.DiskSize = release.DefaultDiskSize

	g.options.Async = true

	if g.options.Stream {
		g.options.Async = false
	}

	gcbSubs, gcbSubsErr := g.SetGCBSubstitutions(toolOrg, toolRepo, toolRef)
	if gcbSubs == nil || gcbSubsErr != nil {
		return gcbSubsErr
	}

	if g.options.NoMock {
		// TODO: Consider a '--yes' flag so we can mock this
		_, nomockSubmit, askErr := util.Ask(
			fmt.Sprintf("Really submit a --nomock release job against the %s branch? (yes/no)", g.options.Branch),
			"yes",
			3,
		)
		if askErr != nil {
			return askErr
		}

		if nomockSubmit {
			gcbSubs["NOMOCK_TAG"] = "nomock"
			gcbSubs["NOMOCK"] = fmt.Sprintf("--%s", gcbSubs["NOMOCK_TAG"])
		}
	} else {
		// TODO: Remove once cloudbuild.yaml doesn't strictly require vars to be set.
		gcbSubs["NOMOCK_TAG"] = ""
		gcbSubs["NOMOCK"] = ""

		bucketPrefix := release.BucketPrefix

		userBucket := fmt.Sprintf("%s%s", bucketPrefix, gcbSubs["GCP_USER_TAG"])
		userBucketSetErr := os.Setenv("USER_BUCKET", userBucket)
		if userBucketSetErr != nil {
			return userBucketSetErr
		}

		testBucket := fmt.Sprintf("%s%s", bucketPrefix, "gcb")
		testBucketSetErr := os.Setenv("BUCKET", testBucket)
		if testBucketSetErr != nil {
			return testBucketSetErr
		}
	}

	toolRoot, err := os.Getwd()
	if err != nil {
		return err
	}

	logrus.Info("Listing GCB substitutions prior to build submission...")
	for k, v := range gcbSubs {
		logrus.Infof("%s: %s", k, v)
	}

	var jobType string
	switch {
	// TODO: Consider a '--validate' flag to validate the GCB config without submitting
	case g.options.Stage:
		jobType = "stage"
	case g.options.Release:
		jobType = "release"
	default:
		return g.listJobs(g.options.Project, g.options.LastJobs)
	}

	gcbSubs["LOG_LEVEL"] = g.options.LogLevel

	g.options.ConfigDir = filepath.Join(toolRoot, "gcb", jobType)
	prepareBuildErr := build.PrepareBuilds(&g.options.Options)
	if prepareBuildErr != nil {
		return prepareBuildErr
	}

	version, err := g.repoClient.GetTag()
	if err != nil {
		return errors.Wrap(err, "getting current tag")
	}

	return errors.Wrap(
		build.RunSingleJob(&g.options.Options, "", "", version, gcbSubs),
		"run GCB job",
	)
}

// SetGCBSubstitutions takes a set of `Options` and returns a map of GCB
// substitutions.
func (g *GCB) SetGCBSubstitutions(toolOrg, toolRepo, toolRef string) (map[string]string, error) {
	gcbSubs := map[string]string{}

	gcbSubs["TOOL_ORG"] = toolOrg
	gcbSubs["TOOL_REPO"] = toolRepo
	gcbSubs["TOOL_REF"] = toolRef

	gcpUser := g.options.GcpUser
	if gcpUser == "" {
		var gcpUserErr error
		gcpUser, gcpUserErr = auth.GetCurrentGCPUser()
		if gcpUserErr != nil {
			return gcbSubs, gcpUserErr
		}
	} else {
		// TODO: Consider removing this once the 'gcloud auth' is testable in CI
		gcpUser = auth.NormalizeGCPUser(gcpUser)
	}

	gcbSubs["GCP_USER_TAG"] = gcpUser

	gcbSubs["TYPE"] = g.options.ReleaseType
	gcbSubs["TYPE_TAG"] = g.options.ReleaseType

	gcbSubs["RELEASE_BRANCH"] = g.options.Branch

	buildVersion := g.options.BuildVersion
	if g.options.Release && buildVersion == "" {
		return gcbSubs, errors.New("Build version must be specified when sending a release GCB run")
	}

	if g.options.Stage && g.options.BuildAtHead {
		hash, err := git.LSRemoteExec(git.GetDefaultKubernetesRepoURL(), "rev-parse", g.options.Branch)
		if err != nil {
			return gcbSubs, errors.Wrapf(
				err, "execute rev-parse for branch %s", g.options.Branch,
			)
		}

		fields := strings.Fields(hash)
		if len(fields) < 1 {
			return gcbSubs, errors.Errorf("unexpected output: %s", hash)
		}

		buildVersion = fields[0]
	}

	if buildVersion == "" {
		var versionErr error
		buildVersion, versionErr = g.versionClient.GetKubeVersionForBranch(
			release.VersionTypeCILatest, g.options.Branch,
		)
		if versionErr != nil {
			return gcbSubs, versionErr
		}
	}

	gcbSubs["BUILDVERSION"] = buildVersion

	kc := kubecross.New()
	kcVersionBranch, err := kc.ForBranch(g.options.Branch)
	if err != nil {
		// If the kubecross version is not set, we will get a 404 from GitHub
		// we do not err but use the latest version
		if !strings.Contains(err.Error(), "404") {
			return gcbSubs, errors.Wrap(err, "retrieve kube-cross version")
		}
		logrus.Infof("KubeCross version not set for %s, falling back to latest", g.options.Branch)
	}

	kcVersionLatest := kcVersionBranch
	if g.options.Branch != git.DefaultBranch {
		kcVersionLatest, err = kc.Latest()
		if err != nil {
			return gcbSubs, errors.Wrap(err, "retrieve latest kube-cross version")
		}

		// if kcVersionBranch is empty, the branch does not exist yet, we use
		// the latest kubecross version
		if kcVersionBranch == "" {
			kcVersionBranch = kcVersionLatest
		}
	}
	gcbSubs["KUBE_CROSS_VERSION"] = kcVersionBranch
	gcbSubs["KUBE_CROSS_VERSION_LATEST"] = kcVersionLatest

	buildVersionSemver, err := util.TagStringToSemver(buildVersion)
	if err != nil {
		return gcbSubs, errors.Wrap(err, "parse build version")
	}

	createBranch, err := g.releaseClient.NeedsCreation(
		g.options.Branch, g.options.ReleaseType, buildVersionSemver,
	)
	if err != nil {
		return nil, errors.Wrap(err, "check if branch needs to be created")
	}
	versions, err := g.releaseClient.GenerateReleaseVersion(
		g.options.ReleaseType, buildVersion,
		g.options.Branch, createBranch,
	)
	if err != nil {
		return nil, errors.Wrap(err, "generate release version")
	}
	primeSemver, err := util.TagStringToSemver(versions.Prime())
	if err != nil {
		return gcbSubs, errors.Wrap(err, "parse prime version")
	}

	gcbSubs["MAJOR_VERSION_TAG"] = strconv.FormatUint(primeSemver.Major, 10)
	gcbSubs["MINOR_VERSION_TAG"] = strconv.FormatUint(primeSemver.Minor, 10)
	gcbSubs["PATCH_VERSION_TAG"] = strconv.FormatUint(primeSemver.Patch, 10)
	gcbSubs["KUBERNETES_VERSION_TAG"] = primeSemver.String()

	return gcbSubs, nil
}

// listJobs lists recent GCB jobs run in the specified project
func (g *GCB) listJobs(project string, lastJobs int64) error {
	if lastJobs < 0 {
		logrus.Infof("--list-jobs was set to a negative number, defaulting to 5")
		lastJobs = 5
	}

	logrus.Infof("Listing last %d GCB jobs:", lastJobs)
	return g.listJobsClient.ListJobs(project, lastJobs)
}
<commit_msg>GCB: Always file on master when failing to retrieve kcross ver<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gcb

//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate

import (
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/blang/semver"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"

	"k8s.io/release/pkg/gcp"
	"k8s.io/release/pkg/gcp/auth"
	"k8s.io/release/pkg/gcp/build"
	"k8s.io/release/pkg/git"
	"k8s.io/release/pkg/kubecross"
	"k8s.io/release/pkg/release"
	"sigs.k8s.io/release-utils/util"
)

// GCB is the main structure of this package.
type GCB struct {
	options        *Options
	repoClient     Repository
	versionClient  Version
	listJobsClient ListJobs
	releaseClient  Release
}

// New creates a new `*GCB` instance.
func New(options *Options) *GCB {
	return &GCB{
		repoClient:     release.NewRepo(),
		versionClient:  release.NewVersion(),
		listJobsClient: &defaultListJobsClient{},
		releaseClient:  &defaultReleaseClient{},
		options:        options,
	}
}

// SetRepoClient can be used to set the internal `Repository` client.
func (g *GCB) SetRepoClient(client Repository) {
	g.repoClient = client
}

// SetVersionClient can be used to set the internal `Version` client.
func (g *GCB) SetVersionClient(client Version) {
	g.versionClient = client
}

// SetListJobsClient can be used to set the internal `ListJobs` client.
func (g *GCB) SetListJobsClient(client ListJobs) {
	g.listJobsClient = client
}

// SetReleaseClient can be used to set the internal `Release` client.
func (g *GCB) SetReleaseClient(client Release) {
	g.releaseClient = client
}

type Options struct {
	build.Options
	NoMock       bool
	Stage        bool
	Release      bool
	Stream       bool
	BuildAtHead  bool
	Branch       string
	ReleaseType  string
	BuildVersion string
	GcpUser      string
	LogLevel     string
	LastJobs     int64
}

// NewDefaultOptions returns a new default `*Options` instance.
func NewDefaultOptions() *Options {
	return &Options{
		LogLevel: logrus.StandardLogger().GetLevel().String(),
		Options:  *build.NewDefaultOptions(),
	}
}

//counterfeiter:generate . Repository
type Repository interface {
	Open() error
	CheckState(string, string, string, bool) error
	GetTag() (string, error)
}

//counterfeiter:generate . Version
type Version interface {
	GetKubeVersionForBranch(release.VersionType, string) (string, error)
}

//counterfeiter:generate . ListJobs
type ListJobs interface {
	ListJobs(project string, lastJobs int64) error
}

type defaultListJobsClient struct{}

func (d *defaultListJobsClient) ListJobs(project string, lastJobs int64) error {
	return build.ListJobs(project, lastJobs)
}

//counterfeiter:generate . Release
type Release interface {
	NeedsCreation(
		branch, releaseType string, buildVersion semver.Version,
	) (createReleaseBranch bool, err error)
	GenerateReleaseVersion(
		releaseType, version, branch string, branchFromMaster bool,
	) (*release.Versions, error)
}

type defaultReleaseClient struct{}

func (*defaultReleaseClient) NeedsCreation(
	branch, releaseType string, buildVersion semver.Version,
) (createReleaseBranch bool, err error) {
	return release.NewBranchChecker().NeedsCreation(
		branch, releaseType, buildVersion,
	)
}

func (*defaultReleaseClient) GenerateReleaseVersion(
	releaseType, version, branch string, branchFromMaster bool,
) (*release.Versions, error) {
	return release.GenerateReleaseVersion(
		releaseType, version, branch, branchFromMaster,
	)
}

// Validate checks if the Options are valid.
func (o *Options) Validate() error {
	if o.Stage && o.Release {
		return errors.New("cannot specify both the 'stage' and 'release' flag; resubmit with only one build type selected")
	}

	if o.Branch == git.DefaultBranch {
		if o.ReleaseType == release.ReleaseTypeRC || o.ReleaseType == release.ReleaseTypeOfficial {
			return errors.Errorf("cannot cut a release candidate or an official release from %s", git.DefaultBranch)
		}
	} else {
		if o.ReleaseType == release.ReleaseTypeAlpha || o.ReleaseType == release.ReleaseTypeBeta {
			return errors.New("cannot cut an alpha or beta release from a release branch")
		}
	}

	if o.BuildVersion != "" && o.BuildAtHead {
		return errors.New("cannot specify both the 'build-version' and 'build-at-head' flag; resubmit with only one build option selected")
	}

	if o.BuildAtHead && o.Release {
		return errors.New("cannot specify both the 'build-at-head' flag together with the 'release' flag; resubmit with a 'build-version' flag set")
	}

	return nil
}

// Submit is the main method responsible for submitting release jobs to GCB.
func (g *GCB) Submit() error {
	if err := g.options.Validate(); err != nil {
		return errors.Wrap(err, "validating GCB options")
	}

	toolOrg := release.GetToolOrg()
	toolRepo := release.GetToolRepo()
	toolRef := release.GetToolRef()

	if err := gcp.PreCheck(); err != nil {
		return errors.Wrap(err, "pre-checking for GCP package usage")
	}

	if err := g.repoClient.Open(); err != nil {
		return errors.Wrap(err, "open release repo")
	}

	if err := g.repoClient.CheckState(toolOrg, toolRepo, toolRef, g.options.NoMock); err != nil {
		return errors.Wrap(err, "verifying repository state")
	}

	logrus.Infof("Running GCB with the following options: %+v", g.options)

	g.options.NoSource = true
	g.options.DiskSize = release.DefaultDiskSize

	g.options.Async = true

	if g.options.Stream {
		g.options.Async = false
	}

	gcbSubs, gcbSubsErr := g.SetGCBSubstitutions(toolOrg, toolRepo, toolRef)
	if gcbSubs == nil || gcbSubsErr != nil {
		return gcbSubsErr
	}

	if g.options.NoMock {
		// TODO: Consider a '--yes' flag so we can mock this
		_, nomockSubmit, askErr := util.Ask(
			fmt.Sprintf("Really submit a --nomock release job against the %s branch? (yes/no)", g.options.Branch),
			"yes",
			3,
		)
		if askErr != nil {
			return askErr
		}

		if nomockSubmit {
			gcbSubs["NOMOCK_TAG"] = "nomock"
			gcbSubs["NOMOCK"] = fmt.Sprintf("--%s", gcbSubs["NOMOCK_TAG"])
		}
	} else {
		// TODO: Remove once cloudbuild.yaml doesn't strictly require vars to be set.
		gcbSubs["NOMOCK_TAG"] = ""
		gcbSubs["NOMOCK"] = ""

		bucketPrefix := release.BucketPrefix

		userBucket := fmt.Sprintf("%s%s", bucketPrefix, gcbSubs["GCP_USER_TAG"])
		userBucketSetErr := os.Setenv("USER_BUCKET", userBucket)
		if userBucketSetErr != nil {
			return userBucketSetErr
		}

		testBucket := fmt.Sprintf("%s%s", bucketPrefix, "gcb")
		testBucketSetErr := os.Setenv("BUCKET", testBucket)
		if testBucketSetErr != nil {
			return testBucketSetErr
		}
	}

	toolRoot, err := os.Getwd()
	if err != nil {
		return err
	}

	logrus.Info("Listing GCB substitutions prior to build submission...")
	for k, v := range gcbSubs {
		logrus.Infof("%s: %s", k, v)
	}

	var jobType string
	switch {
	// TODO: Consider a '--validate' flag to validate the GCB config without submitting
	case g.options.Stage:
		jobType = "stage"
	case g.options.Release:
		jobType = "release"
	default:
		return g.listJobs(g.options.Project, g.options.LastJobs)
	}

	gcbSubs["LOG_LEVEL"] = g.options.LogLevel

	g.options.ConfigDir = filepath.Join(toolRoot, "gcb", jobType)
	prepareBuildErr := build.PrepareBuilds(&g.options.Options)
	if prepareBuildErr != nil {
		return prepareBuildErr
	}

	version, err := g.repoClient.GetTag()
	if err != nil {
		return errors.Wrap(err, "getting current tag")
	}

	return errors.Wrap(
		build.RunSingleJob(&g.options.Options, "", "", version, gcbSubs),
		"run GCB job",
	)
}

// SetGCBSubstitutions takes a set of `Options` and returns a map of GCB
// substitutions.
func (g *GCB) SetGCBSubstitutions(toolOrg, toolRepo, toolRef string) (map[string]string, error) {
	gcbSubs := map[string]string{}

	gcbSubs["TOOL_ORG"] = toolOrg
	gcbSubs["TOOL_REPO"] = toolRepo
	gcbSubs["TOOL_REF"] = toolRef

	gcpUser := g.options.GcpUser
	if gcpUser == "" {
		var gcpUserErr error
		gcpUser, gcpUserErr = auth.GetCurrentGCPUser()
		if gcpUserErr != nil {
			return gcbSubs, gcpUserErr
		}
	} else {
		// TODO: Consider removing this once the 'gcloud auth' is testable in CI
		gcpUser = auth.NormalizeGCPUser(gcpUser)
	}

	gcbSubs["GCP_USER_TAG"] = gcpUser

	gcbSubs["TYPE"] = g.options.ReleaseType
	gcbSubs["TYPE_TAG"] = g.options.ReleaseType

	gcbSubs["RELEASE_BRANCH"] = g.options.Branch

	buildVersion := g.options.BuildVersion
	if g.options.Release && buildVersion == "" {
		return gcbSubs, errors.New("Build version must be specified when sending a release GCB run")
	}

	if g.options.Stage && g.options.BuildAtHead {
		hash, err := git.LSRemoteExec(git.GetDefaultKubernetesRepoURL(), "rev-parse", g.options.Branch)
		if err != nil {
			return gcbSubs, errors.Wrapf(
				err, "execute rev-parse for branch %s", g.options.Branch,
			)
		}

		fields := strings.Fields(hash)
		if len(fields) < 1 {
			return gcbSubs, errors.Errorf("unexpected output: %s", hash)
		}

		buildVersion = fields[0]
	}

	if buildVersion == "" {
		var versionErr error
		buildVersion, versionErr = g.versionClient.GetKubeVersionForBranch(
			release.VersionTypeCILatest, g.options.Branch,
		)
		if versionErr != nil {
			return gcbSubs, versionErr
		}
	}

	gcbSubs["BUILDVERSION"] = buildVersion

	kc := kubecross.New()
	kcVersionBranch, err := kc.ForBranch(g.options.Branch)
	if err != nil {
		// If the kubecross version is not set, we will get a 404 from GitHub.
		// In that case, we do not err but use the latest version (unless we're on main branch)
		if g.options.Branch == git.DefaultBranch || !strings.Contains(err.Error(), "404") {
			return gcbSubs, errors.Wrap(err, "retrieve kube-cross version")
		}
		logrus.Infof("KubeCross version not set for %s, falling back to latest", g.options.Branch)
	}

	kcVersionLatest := kcVersionBranch
	if g.options.Branch != git.DefaultBranch {
		kcVersionLatest, err = kc.Latest()
		if err != nil {
			return gcbSubs, errors.Wrap(err, "retrieve latest kube-cross version")
		}

		// if kcVersionBranch is empty, the branch does not exist yet, we use
		// the latest kubecross version
		if kcVersionBranch == "" {
			kcVersionBranch = kcVersionLatest
		}
	}
	gcbSubs["KUBE_CROSS_VERSION"] = kcVersionBranch
	gcbSubs["KUBE_CROSS_VERSION_LATEST"] = kcVersionLatest

	buildVersionSemver, err := util.TagStringToSemver(buildVersion)
	if err != nil {
		return gcbSubs, errors.Wrap(err, "parse build version")
	}

	createBranch, err := g.releaseClient.NeedsCreation(
		g.options.Branch, g.options.ReleaseType, buildVersionSemver,
	)
	if err != nil {
		return nil, errors.Wrap(err, "check if branch needs to be created")
	}
	versions, err := g.releaseClient.GenerateReleaseVersion(
		g.options.ReleaseType, buildVersion,
		g.options.Branch, createBranch,
	)
	if err != nil {
		return nil, errors.Wrap(err, "generate release version")
	}
	primeSemver, err := util.TagStringToSemver(versions.Prime())
	if err != nil {
		return gcbSubs, errors.Wrap(err, "parse prime version")
	}

	gcbSubs["MAJOR_VERSION_TAG"] = strconv.FormatUint(primeSemver.Major, 10)
	gcbSubs["MINOR_VERSION_TAG"] = strconv.FormatUint(primeSemver.Minor, 10)
	gcbSubs["PATCH_VERSION_TAG"] = strconv.FormatUint(primeSemver.Patch, 10)
	gcbSubs["KUBERNETES_VERSION_TAG"] = primeSemver.String()

	return gcbSubs, nil
}

// listJobs lists recent GCB jobs run in the specified project
func (g *GCB) listJobs(project string, lastJobs int64) error {
	if lastJobs < 0 {
		logrus.Infof("--list-jobs was set to a negative number, defaulting to 5")
		lastJobs = 5
	}

	logrus.Infof("Listing last %d GCB jobs:", lastJobs)
	return g.listJobsClient.ListJobs(project, lastJobs)
}
<|endoftext|>


<commit_before><commit_msg>only user can twitter and update user data<commit_after><|endoftext|>


<commit_before>// Copyright 2013 The Go Circuit Project
// Use of this source code is governed by the license for
// The Go Circuit Project, found in the LICENSE file.
//
// Authors:
//   2014 Petar Maymounkov <p@gocircuit.org>

package proc

import (
	"bytes"

	"github.com/gocircuit/circuit/kit/iomisc"
	"github.com/gocircuit/circuit/kit/fs/rh"
	"github.com/gocircuit/circuit/kit/fs/namespace/file"
)

type WaitFile struct {
	p *Proc
}

func NewWaitFile(p *Proc) file.File {
	return &WaitFile{p: p}
}

func (f *WaitFile) Perm() rh.Perm {
	return 0444 // r--r--r--
}

func (f *WaitFile) Open(flag rh.Flag, intr rh.Intr) (rh.FID, error) {
	if flag.Attr != rh.ReadOnly {
		return nil, rh.ErrPerm
	}
	stat, err := f.p.Wait(intr)
	if err == rh.ErrIntr {
		return nil, err
	}
	return file.NewOpenReaderFile(iomisc.ReaderNopCloser(bytes.NewBufferString(stat.String()))), nil
}

func (f *WaitFile) Remove() error {
	return rh.ErrPerm
}
<commit_msg>proc wait blocks on first read (as opposed to on open)<commit_after>// Copyright 2013 The Go Circuit Project
// Use of this source code is governed by the license for
// The Go Circuit Project, found in the LICENSE file.
//
// Authors:
//   2014 Petar Maymounkov <p@gocircuit.org>

package proc

import (
	"bytes"

	"github.com/gocircuit/circuit/kit/iomisc"
	"github.com/gocircuit/circuit/kit/fs/rh"
	"github.com/gocircuit/circuit/kit/fs/namespace/file"
)

type WaitFile struct {
	p *Proc
}

func NewWaitFile(p *Proc) file.File {
	return &WaitFile{p: p}
}

func (f *WaitFile) Perm() rh.Perm {
	return 0444 // r--r--r--
}

func (f *WaitFile) Open(flag rh.Flag, intr rh.Intr) (rh.FID, error) {
	if flag.Attr != rh.ReadOnly {
		return nil, rh.ErrPerm
	}
	return file.NewCommitFile(
		func (intr rh.Intr) (rh.FID, error) {
			stat, err := f.p.Wait(intr)
			if err == rh.ErrIntr {
				return nil, err
			}
			return file.NewOpenReaderFile(iomisc.ReaderNopCloser(bytes.NewBufferString(stat.String()))), nil
		},
	), nil
}

func (f *WaitFile) Remove() error {
	return rh.ErrPerm
}
<|endoftext|>


<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package common

import (
	"errors"
	"fmt"
	"log"
	"org/apache/htrace/conf"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"
)

// A logSink is a place logs can be written to.
type logSink struct {
	path     logPath
	file     *os.File
	lock     sync.Mutex
	refCount int // protected by logFilesLock
}

// Write to the logSink.
func (sink *logSink) write(str string) {
	sink.lock.Lock()
	defer sink.lock.Unlock()
	_, err := sink.file.Write([]byte(str))
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error logging to '%s': %s\n", sink.path, err.Error())
	}
}

// Unreference the logSink.  If there are no more references, and the logSink is
// closeable, then we will close it here.
func (sink *logSink) Unref() {
	logFilesLock.Lock()
	defer logFilesLock.Unlock()
	sink.refCount--
	if sink.refCount <= 0 {
		if sink.path.IsCloseable() {
			err := sink.file.Close()
			if err != nil {
				fmt.Fprintf(os.Stderr, "Error closing log file %s: %s\n",
					sink.path, err.Error())
			}
		}
		logSinks[sink.path] = nil
	}
}

type logPath string

// An empty LogPath represents "stdout."
const STDOUT_LOG_PATH = ""

// Convert a path to a logPath.
func logPathFromString(path string) logPath {
	if path == STDOUT_LOG_PATH {
		return logPath("")
	}
	absPath, err := filepath.Abs(path)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to get absolute path of %s: %s\n",
			path, err.Error())
		return logPath(path)
	}
	return logPath(absPath)
}

// Convert the path to a human-readable string.
func (path logPath) String() string {
	if path == "" {
		return "(stdout)"
	} else {
		return string(path)
	}
}

// Return true if the path is closeable.  stdout is not closeable.
func (path logPath) IsCloseable() bool {
	return path != STDOUT_LOG_PATH
}

func (path logPath) Open() *logSink {
	if path == STDOUT_LOG_PATH {
		return &logSink{path: path, file: os.Stdout}
	}
	file, err := os.OpenFile(string(path), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
	if err != nil {
		sink := &logSink{path: STDOUT_LOG_PATH, file: os.Stdout}
		fmt.Fprintf(os.Stderr, "Failed to open log file %s: %s\n",
			path, err.Error())
		return sink
	}
	return &logSink{path: path, file: file}
}

var logFilesLock sync.Mutex

var logSinks map[logPath]*logSink = make(map[logPath]*logSink)

func getOrCreateLogSink(pathStr string) *logSink {
	path := logPathFromString(pathStr)
	logFilesLock.Lock()
	defer logFilesLock.Unlock()
	sink := logSinks[path]
	if sink == nil {
		sink = path.Open()
		logSinks[path] = sink
	}
	sink.refCount++
	return sink
}

type Level int

const (
	TRACE Level = iota
	DEBUG
	INFO
	WARN
	ERROR
)

var levelToString map[Level]string = map[Level]string{
	TRACE: "TRACE",
	DEBUG: "DEBUG",
	INFO:  "INFO",
	WARN:  "WARN",
	ERROR: "ERROR",
}

func (level Level) String() string {
	return levelToString[level]
}

func (level Level) LogString() string {
	return level.String()[0:1]
}

func LevelFromString(str string) (Level, error) {
	for k, v := range levelToString {
		if strings.ToLower(v) == strings.ToLower(str) {
			return k, nil
		}
	}
	var levelNames sort.StringSlice
	levelNames = make([]string, len(levelToString))
	var i int
	for _, v := range levelToString {
		levelNames[i] = v
		i++
	}
	sort.Sort(levelNames)
	return TRACE, errors.New(fmt.Sprintf("No such level as '%s'.  Valid "+
		"levels are '%v'\n", str, levelNames))
}

type Logger struct {
	sink  *logSink
	Level Level
}

func NewLogger(faculty string, cnf *conf.Config) *Logger {
	path, level := parseConf(faculty, cnf)
	sink := getOrCreateLogSink(path)
	return &Logger{sink: sink, Level: level}
}

func parseConf(faculty string, cnf *conf.Config) (string, Level) {
	facultyLogPathKey := faculty + "." + conf.HTRACE_LOG_PATH
	var facultyLogPath string
	if cnf.Contains(facultyLogPathKey) {
		facultyLogPath = cnf.Get(facultyLogPathKey)
	} else {
		facultyLogPath = cnf.Get(conf.HTRACE_LOG_PATH)
	}
	facultyLogLevelKey := faculty + "." + conf.HTRACE_LOG_LEVEL
	var facultyLogLevelStr string
	if cnf.Contains(facultyLogLevelKey) {
		facultyLogLevelStr = cnf.Get(facultyLogLevelKey)
	} else {
		facultyLogLevelStr = cnf.Get(conf.HTRACE_LOG_LEVEL)
	}
	level, err := LevelFromString(facultyLogLevelStr)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error configuring log level: %s.  Using TRACE.\n")
		level = TRACE
	}
	return facultyLogPath, level
}

func (lg *Logger) Trace(str string) {
	lg.Write(TRACE, str)
}

func (lg *Logger) Tracef(format string, v ...interface{}) {
	lg.Write(TRACE, fmt.Sprintf(format, v...))
}

func (lg *Logger) Debug(str string) {
	lg.Write(DEBUG, str)
}

func (lg *Logger) Debugf(format string, v ...interface{}) {
	lg.Write(DEBUG, fmt.Sprintf(format, v...))
}

func (lg *Logger) Info(str string) {
	lg.Write(INFO, str)
}

func (lg *Logger) Infof(format string, v ...interface{}) {
	lg.Write(INFO, fmt.Sprintf(format, v...))
}

func (lg *Logger) Warn(str string) error {
	lg.Write(WARN, str)
	return errors.New(str)
}

func (lg *Logger) Warnf(format string, v ...interface{}) error {
	str := fmt.Sprintf(format, v...)
	lg.Write(WARN, str)
	return errors.New(str)
}

func (lg *Logger) Error(str string) error {
	lg.Write(ERROR, str)
	return errors.New(str)
}

func (lg *Logger) Errorf(format string, v ...interface{}) error {
	str := fmt.Sprintf(format, v...)
	lg.Write(ERROR, str)
	return errors.New(str)
}

func (lg *Logger) Write(level Level, str string) {
	if level >= lg.Level {
		lg.sink.write(time.Now().Format(time.RFC3339) + " " +
			level.LogString() + ": " + str)
	}
}

//
// A few functions which can be used to determine if a certain level of tracing
// is enabled.  These are useful in situations when evaluating the parameters
// of a logging function is expensive.  (Note, however, that we don't pay the
// cost of string concatenation and manipulation when a log message doesn't
// trigger.)
//

func (lg *Logger) TraceEnabled() bool {
	return lg.Level <= TRACE
}

func (lg *Logger) DebugEnabled() bool {
	return lg.Level <= DEBUG
}

func (lg *Logger) InfoEnabled() bool {
	return lg.Level <= INFO
}

func (lg *Logger) WarnEnabled() bool {
	return lg.Level <= WARN
}

func (lg *Logger) ErrorEnabled() bool {
	return lg.Level <= ERROR
}

func (lg *Logger) LevelEnabled(level Level) bool {
	return lg.Level <= level
}

func (lg *Logger) Close() {
	lg.sink.Unref()
	lg.sink = nil
}

// Wraps an htrace logger in a golang standard logger.
//
// This is a bit messy because of the difference in interfaces between the
// golang standard logger and the htrace logger.  The golang standard logger
// doesn't support log levels directly, so you must choose up front what htrace
// log level all messages should be treated as.  Golang standard loggers expect
// to be able to write to an io.Writer, but make no guarantees about whether
// they will break messages into multiple Write() calls (although this does
// not seem to be a major problem in practice.)
//
// Despite these limitations, it's still useful to have this method to be able
// to log things that come out of the go HTTP server and other standard library
// systems.
type WrappedLogger struct {
	lg    *Logger
	level Level
}

func (lg *Logger) Wrap(prefix string, level Level) *log.Logger {
	wlg := &WrappedLogger{
		lg:    lg,
		level: level,
	}
	return log.New(wlg, prefix, 0)
}

func (wlg *WrappedLogger) Write(p []byte) (int, error) {
	wlg.lg.Write(wlg.level, string(p))
	return len(p), nil
}
<commit_msg>HTRACE-306. htraced: logs should use UTC (Colin Patrick McCabe via iwasakims)<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package common

import (
	"errors"
	"fmt"
	"log"
	"org/apache/htrace/conf"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"
)

// A logSink is a place logs can be written to.
type logSink struct {
	path     logPath
	file     *os.File
	lock     sync.Mutex
	refCount int // protected by logFilesLock
}

// Write to the logSink.
func (sink *logSink) write(str string) {
	sink.lock.Lock()
	defer sink.lock.Unlock()
	_, err := sink.file.Write([]byte(str))
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error logging to '%s': %s\n", sink.path, err.Error())
	}
}

// Unreference the logSink.  If there are no more references, and the logSink is
// closeable, then we will close it here.
func (sink *logSink) Unref() {
	logFilesLock.Lock()
	defer logFilesLock.Unlock()
	sink.refCount--
	if sink.refCount <= 0 {
		if sink.path.IsCloseable() {
			err := sink.file.Close()
			if err != nil {
				fmt.Fprintf(os.Stderr, "Error closing log file %s: %s\n",
					sink.path, err.Error())
			}
		}
		logSinks[sink.path] = nil
	}
}

type logPath string

// An empty LogPath represents "stdout."
const STDOUT_LOG_PATH = ""

// Convert a path to a logPath.
func logPathFromString(path string) logPath {
	if path == STDOUT_LOG_PATH {
		return logPath("")
	}
	absPath, err := filepath.Abs(path)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to get absolute path of %s: %s\n",
			path, err.Error())
		return logPath(path)
	}
	return logPath(absPath)
}

// Convert the path to a human-readable string.
func (path logPath) String() string {
	if path == "" {
		return "(stdout)"
	} else {
		return string(path)
	}
}

// Return true if the path is closeable.  stdout is not closeable.
func (path logPath) IsCloseable() bool {
	return path != STDOUT_LOG_PATH
}

func (path logPath) Open() *logSink {
	if path == STDOUT_LOG_PATH {
		return &logSink{path: path, file: os.Stdout}
	}
	file, err := os.OpenFile(string(path), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
	if err != nil {
		sink := &logSink{path: STDOUT_LOG_PATH, file: os.Stdout}
		fmt.Fprintf(os.Stderr, "Failed to open log file %s: %s\n",
			path, err.Error())
		return sink
	}
	return &logSink{path: path, file: file}
}

var logFilesLock sync.Mutex

var logSinks map[logPath]*logSink = make(map[logPath]*logSink)

func getOrCreateLogSink(pathStr string) *logSink {
	path := logPathFromString(pathStr)
	logFilesLock.Lock()
	defer logFilesLock.Unlock()
	sink := logSinks[path]
	if sink == nil {
		sink = path.Open()
		logSinks[path] = sink
	}
	sink.refCount++
	return sink
}

type Level int

const (
	TRACE Level = iota
	DEBUG
	INFO
	WARN
	ERROR
)

var levelToString map[Level]string = map[Level]string{
	TRACE: "TRACE",
	DEBUG: "DEBUG",
	INFO:  "INFO",
	WARN:  "WARN",
	ERROR: "ERROR",
}

func (level Level) String() string {
	return levelToString[level]
}

func (level Level) LogString() string {
	return level.String()[0:1]
}

func LevelFromString(str string) (Level, error) {
	for k, v := range levelToString {
		if strings.ToLower(v) == strings.ToLower(str) {
			return k, nil
		}
	}
	var levelNames sort.StringSlice
	levelNames = make([]string, len(levelToString))
	var i int
	for _, v := range levelToString {
		levelNames[i] = v
		i++
	}
	sort.Sort(levelNames)
	return TRACE, errors.New(fmt.Sprintf("No such level as '%s'.  Valid "+
		"levels are '%v'\n", str, levelNames))
}

type Logger struct {
	sink  *logSink
	Level Level
}

func NewLogger(faculty string, cnf *conf.Config) *Logger {
	path, level := parseConf(faculty, cnf)
	sink := getOrCreateLogSink(path)
	return &Logger{sink: sink, Level: level}
}

func parseConf(faculty string, cnf *conf.Config) (string, Level) {
	facultyLogPathKey := faculty + "." + conf.HTRACE_LOG_PATH
	var facultyLogPath string
	if cnf.Contains(facultyLogPathKey) {
		facultyLogPath = cnf.Get(facultyLogPathKey)
	} else {
		facultyLogPath = cnf.Get(conf.HTRACE_LOG_PATH)
	}
	facultyLogLevelKey := faculty + "." + conf.HTRACE_LOG_LEVEL
	var facultyLogLevelStr string
	if cnf.Contains(facultyLogLevelKey) {
		facultyLogLevelStr = cnf.Get(facultyLogLevelKey)
	} else {
		facultyLogLevelStr = cnf.Get(conf.HTRACE_LOG_LEVEL)
	}
	level, err := LevelFromString(facultyLogLevelStr)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error configuring log level: %s.  Using TRACE.\n")
		level = TRACE
	}
	return facultyLogPath, level
}

func (lg *Logger) Trace(str string) {
	lg.Write(TRACE, str)
}

func (lg *Logger) Tracef(format string, v ...interface{}) {
	lg.Write(TRACE, fmt.Sprintf(format, v...))
}

func (lg *Logger) Debug(str string) {
	lg.Write(DEBUG, str)
}

func (lg *Logger) Debugf(format string, v ...interface{}) {
	lg.Write(DEBUG, fmt.Sprintf(format, v...))
}

func (lg *Logger) Info(str string) {
	lg.Write(INFO, str)
}

func (lg *Logger) Infof(format string, v ...interface{}) {
	lg.Write(INFO, fmt.Sprintf(format, v...))
}

func (lg *Logger) Warn(str string) error {
	lg.Write(WARN, str)
	return errors.New(str)
}

func (lg *Logger) Warnf(format string, v ...interface{}) error {
	str := fmt.Sprintf(format, v...)
	lg.Write(WARN, str)
	return errors.New(str)
}

func (lg *Logger) Error(str string) error {
	lg.Write(ERROR, str)
	return errors.New(str)
}

func (lg *Logger) Errorf(format string, v ...interface{}) error {
	str := fmt.Sprintf(format, v...)
	lg.Write(ERROR, str)
	return errors.New(str)
}

func (lg *Logger) Write(level Level, str string) {
	if level >= lg.Level {
		lg.sink.write(time.Now().UTC().Format(time.RFC3339) + " " +
			level.LogString() + ": " + str)
	}
}

//
// A few functions which can be used to determine if a certain level of tracing
// is enabled.  These are useful in situations when evaluating the parameters
// of a logging function is expensive.  (Note, however, that we don't pay the
// cost of string concatenation and manipulation when a log message doesn't
// trigger.)
//

func (lg *Logger) TraceEnabled() bool {
	return lg.Level <= TRACE
}

func (lg *Logger) DebugEnabled() bool {
	return lg.Level <= DEBUG
}

func (lg *Logger) InfoEnabled() bool {
	return lg.Level <= INFO
}

func (lg *Logger) WarnEnabled() bool {
	return lg.Level <= WARN
}

func (lg *Logger) ErrorEnabled() bool {
	return lg.Level <= ERROR
}

func (lg *Logger) LevelEnabled(level Level) bool {
	return lg.Level <= level
}

func (lg *Logger) Close() {
	lg.sink.Unref()
	lg.sink = nil
}

// Wraps an htrace logger in a golang standard logger.
//
// This is a bit messy because of the difference in interfaces between the
// golang standard logger and the htrace logger.  The golang standard logger
// doesn't support log levels directly, so you must choose up front what htrace
// log level all messages should be treated as.  Golang standard loggers expect
// to be able to write to an io.Writer, but make no guarantees about whether
// they will break messages into multiple Write() calls (although this does
// not seem to be a major problem in practice.)
//
// Despite these limitations, it's still useful to have this method to be able
// to log things that come out of the go HTTP server and other standard library
// systems.
type WrappedLogger struct {
	lg    *Logger
	level Level
}

func (lg *Logger) Wrap(prefix string, level Level) *log.Logger {
	wlg := &WrappedLogger{
		lg:    lg,
		level: level,
	}
	return log.New(wlg, prefix, 0)
}

func (wlg *WrappedLogger) Write(p []byte) (int, error) {
	wlg.lg.Write(wlg.level, string(p))
	return len(p), nil
}
<|endoftext|>


<commit_before>// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0

package filters

import (
	"fmt"

	"sigs.k8s.io/kustomize/kyaml/kio"
	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
	"sigs.k8s.io/kustomize/kyaml/yaml"
	"sigs.k8s.io/kustomize/kyaml/yaml/merge3"
)

const (
	mergeSourceAnnotation = "config.kubernetes.io/merge-source"
	mergeSourceOriginal   = "original"
	mergeSourceUpdated    = "updated"
	mergeSourceDest       = "dest"
)

// ResourceMatcher interface is used to match two resources based on IsSameResource implementation
// This is the way to group same logical resources in upstream, local and origin for merge
// The default way to group them is using GVKNN similar to how kubernetes server identifies resources
// Users of this library might have their own interpretation of grouping similar resources
// for e.g. if consumer adds a name-prefix to local resource, it should not be treated as new resource
// for updates etc.
// Hence, the callers of this library may pass different implementation for IsSameResource
type ResourceMatcher interface {
	IsSameResource(node1, node2 *yaml.RNode) bool
}

// ResourceMergeStrategy is the return type from the Handle function in the
// ResourceHandler interface. It determines which version of a resource should
// be included in the output (if any).
type ResourceMergeStrategy int

const (
	// Merge means the output to dest should be the 3-way merge of original,
	// updated and dest.
	Merge ResourceMergeStrategy = iota
	// KeepDest means the version of the resource in dest should be the output.
	KeepDest
	// KeepUpdated means the version of the resource in updated should be the
	// output.
	KeepUpdated
	// KeepOriginal means the version of the resource in original should be the
	// output.
	KeepOriginal
	// Skip means the resource should not be included in the output.
	Skip
)

// ResourceHandler interface is used to determine what should be done for a
// resource once the versions in original, updated and dest has been
// identified based on the ResourceMatcher. This allows users to customize
// what should be the result in dest if a resource has been deleted from
// upstream.
type ResourceHandler interface {
	Handle(original, updated, dest *yaml.RNode) ResourceMergeStrategy
}

// Merge3 performs a 3-way merge on the original, updated, and destination packages.
type Merge3 struct {
	OriginalPath   string
	UpdatedPath    string
	DestPath       string
	MatchFilesGlob []string
	Matcher        ResourceMatcher
	Handler        ResourceHandler
}

func (m Merge3) Merge() error {
	// Read the destination package.  The ReadWriter will take take of deleting files
	// for removed resources.
	var inputs []kio.Reader
	dest := &kio.LocalPackageReadWriter{
		PackagePath:    m.DestPath,
		MatchFilesGlob: m.MatchFilesGlob,
		SetAnnotations: map[string]string{mergeSourceAnnotation: mergeSourceDest},
	}
	inputs = append(inputs, dest)

	// Read the original package
	inputs = append(inputs, kio.LocalPackageReader{
		PackagePath:    m.OriginalPath,
		MatchFilesGlob: m.MatchFilesGlob,
		SetAnnotations: map[string]string{mergeSourceAnnotation: mergeSourceOriginal},
	})

	// Read the updated package
	inputs = append(inputs, kio.LocalPackageReader{
		PackagePath:    m.UpdatedPath,
		MatchFilesGlob: m.MatchFilesGlob,
		SetAnnotations: map[string]string{mergeSourceAnnotation: mergeSourceUpdated},
	})

	return kio.Pipeline{
		Inputs:  inputs,
		Filters: []kio.Filter{m},
		Outputs: []kio.Writer{dest},
	}.Execute()
}

// Filter combines Resources with the same GVK + N + NS into tuples, and then merges them
func (m Merge3) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
	// index the nodes by their identity
	matcher := m.Matcher
	if matcher == nil {
		matcher = &DefaultGVKNNMatcher{MergeOnPath: true}
	}
	handler := m.Handler
	if handler == nil {
		handler = &DefaultResourceHandler{}
	}

	tl := tuples{matcher: matcher}
	for i := range nodes {
		if err := tl.add(nodes[i]); err != nil {
			return nil, err
		}
	}

	// iterate over the inputs, merging as needed
	var output []*yaml.RNode
	for i := range tl.list {
		t := tl.list[i]
		strategy := handler.Handle(t.original, t.updated, t.dest)
		switch strategy {
		case Merge:
			node, err := t.merge()
			if err != nil {
				return nil, err
			}
			if node != nil {
				output = append(output, node)
			}
		case KeepDest:
			output = append(output, t.dest)
		case KeepUpdated:
			output = append(output, t.updated)
		case KeepOriginal:
			output = append(output, t.original)
		case Skip:
			// do nothing
		}
	}
	return output, nil
}

// tuples combines nodes with the same GVK + N + NS
type tuples struct {
	list []*tuple

	// matcher matches the resources for merge
	matcher ResourceMatcher
}

// DefaultGVKNNMatcher holds the default matching of resources implementation based on
// Group, Version, Kind, Name and Namespace of the resource
type DefaultGVKNNMatcher struct {
	// MergeOnPath will use the relative filepath as part of the merge key.
	// This may be necessary if the directory contains multiple copies of
	// the same resource, or resources patches.
	MergeOnPath bool
}

// IsSameResource returns true if metadata of node1 and metadata of node2 belongs to same logical resource
func (dm *DefaultGVKNNMatcher) IsSameResource(node1, node2 *yaml.RNode) bool {
	if node1 == nil || node2 == nil {
		return false
	}

	meta1, err := node1.GetMeta()
	if err != nil {
		return false
	}

	meta2, err := node2.GetMeta()
	if err != nil {
		return false
	}

	if meta1.Name != meta2.Name {
		return false
	}
	if meta1.Namespace != meta2.Namespace {
		return false
	}
	if meta1.APIVersion != meta2.APIVersion {
		return false
	}
	if meta1.Kind != meta2.Kind {
		return false
	}
	if dm.MergeOnPath {
		// directories may contain multiple copies of a resource with the same
		// name, namespace, apiVersion and kind -- e.g. kustomize patches, or
		// multiple environments
		// mergeOnPath configures the merge logic to use the path as part of the
		// resource key
		if meta1.Annotations[kioutil.PathAnnotation] != meta2.Annotations[kioutil.PathAnnotation] {
			return false
		}
	}
	return true
}

// add adds a node to the list, combining it with an existing matching Resource if found
func (ts *tuples) add(node *yaml.RNode) error {
	for i := range ts.list {
		t := ts.list[i]
		if ts.matcher.IsSameResource(addedNode(t), node) {
			return t.add(node)
		}
	}
	t := &tuple{}
	if err := t.add(node); err != nil {
		return err
	}
	ts.list = append(ts.list, t)
	return nil
}

// addedNode returns one on the existing added nodes in the tuple
func addedNode(t *tuple) *yaml.RNode {
	if t.updated != nil {
		return t.updated
	}
	if t.original != nil {
		return t.original
	}
	return t.dest
}

// tuple wraps an original, updated, and dest tuple for a given Resource
type tuple struct {
	original *yaml.RNode
	updated  *yaml.RNode
	dest     *yaml.RNode
}

// add sets the corresponding tuple field for the node
func (t *tuple) add(node *yaml.RNode) error {
	meta, err := node.GetMeta()
	if err != nil {
		return err
	}
	switch meta.Annotations[mergeSourceAnnotation] {
	case mergeSourceDest:
		if t.dest != nil {
			return duplicateError("local", meta.Annotations[kioutil.PathAnnotation])
		}
		t.dest = node
	case mergeSourceOriginal:
		if t.original != nil {
			return duplicateError("original upstream", meta.Annotations[kioutil.PathAnnotation])
		}
		t.original = node
	case mergeSourceUpdated:
		if t.updated != nil {
			return duplicateError("updated upstream", meta.Annotations[kioutil.PathAnnotation])
		}
		t.updated = node
	default:
		return fmt.Errorf("no source annotation for Resource")
	}
	return nil
}

// merge performs a 3-way merge on the tuple
func (t *tuple) merge() (*yaml.RNode, error) {
	return merge3.Merge(t.dest, t.original, t.updated)
}

// duplicateError returns duplicate resources error
func duplicateError(source, filePath string) error {
	return fmt.Errorf(`found duplicate %q resources in file %q, please refer to "update" documentation for the fix`, source, filePath)
}

// DefaultResourceHandler is the default implementation of the ResourceHandler
// interface. It uses the following rules:
// * Keep dest if resource only exists in dest.
// * Keep updated if resource added in updated.
// * Delete dest if updated has been deleted.
// * Don't add the resource back if removed from dest.
// * Otherwise merge.
type DefaultResourceHandler struct{}

func (*DefaultResourceHandler) Handle(original, updated, dest *yaml.RNode) ResourceMergeStrategy {
	switch {
	case original == nil && updated == nil && dest != nil:
		// added locally -- keep dest
		return KeepDest
	case updated != nil && dest == nil:
		// added in the update -- add update
		return KeepUpdated
	case original != nil && updated == nil:
		// deleted in the update
		return Skip
	case original != nil && dest == nil:
		// deleted locally
		return Skip
	default:
		// dest and updated are non-nil -- merge them
		return Merge
	}
}
<commit_msg>Allow the Handle function of the ResourceHandler interface to return an error<commit_after>// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0

package filters

import (
	"fmt"

	"sigs.k8s.io/kustomize/kyaml/kio"
	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
	"sigs.k8s.io/kustomize/kyaml/yaml"
	"sigs.k8s.io/kustomize/kyaml/yaml/merge3"
)

const (
	mergeSourceAnnotation = "config.kubernetes.io/merge-source"
	mergeSourceOriginal   = "original"
	mergeSourceUpdated    = "updated"
	mergeSourceDest       = "dest"
)

// ResourceMatcher interface is used to match two resources based on IsSameResource implementation
// This is the way to group same logical resources in upstream, local and origin for merge
// The default way to group them is using GVKNN similar to how kubernetes server identifies resources
// Users of this library might have their own interpretation of grouping similar resources
// for e.g. if consumer adds a name-prefix to local resource, it should not be treated as new resource
// for updates etc.
// Hence, the callers of this library may pass different implementation for IsSameResource
type ResourceMatcher interface {
	IsSameResource(node1, node2 *yaml.RNode) bool
}

// ResourceMergeStrategy is the return type from the Handle function in the
// ResourceHandler interface. It determines which version of a resource should
// be included in the output (if any).
type ResourceMergeStrategy int

const (
	// Merge means the output to dest should be the 3-way merge of original,
	// updated and dest.
	Merge ResourceMergeStrategy = iota
	// KeepDest means the version of the resource in dest should be the output.
	KeepDest
	// KeepUpdated means the version of the resource in updated should be the
	// output.
	KeepUpdated
	// KeepOriginal means the version of the resource in original should be the
	// output.
	KeepOriginal
	// Skip means the resource should not be included in the output.
	Skip
)

// ResourceHandler interface is used to determine what should be done for a
// resource once the versions in original, updated and dest has been
// identified based on the ResourceMatcher. This allows users to customize
// what should be the result in dest if a resource has been deleted from
// upstream.
type ResourceHandler interface {
	Handle(original, updated, dest *yaml.RNode) (ResourceMergeStrategy, error)
}

// Merge3 performs a 3-way merge on the original, updated, and destination packages.
type Merge3 struct {
	OriginalPath   string
	UpdatedPath    string
	DestPath       string
	MatchFilesGlob []string
	Matcher        ResourceMatcher
	Handler        ResourceHandler
}

func (m Merge3) Merge() error {
	// Read the destination package.  The ReadWriter will take take of deleting files
	// for removed resources.
	var inputs []kio.Reader
	dest := &kio.LocalPackageReadWriter{
		PackagePath:    m.DestPath,
		MatchFilesGlob: m.MatchFilesGlob,
		SetAnnotations: map[string]string{mergeSourceAnnotation: mergeSourceDest},
	}
	inputs = append(inputs, dest)

	// Read the original package
	inputs = append(inputs, kio.LocalPackageReader{
		PackagePath:    m.OriginalPath,
		MatchFilesGlob: m.MatchFilesGlob,
		SetAnnotations: map[string]string{mergeSourceAnnotation: mergeSourceOriginal},
	})

	// Read the updated package
	inputs = append(inputs, kio.LocalPackageReader{
		PackagePath:    m.UpdatedPath,
		MatchFilesGlob: m.MatchFilesGlob,
		SetAnnotations: map[string]string{mergeSourceAnnotation: mergeSourceUpdated},
	})

	return kio.Pipeline{
		Inputs:  inputs,
		Filters: []kio.Filter{m},
		Outputs: []kio.Writer{dest},
	}.Execute()
}

// Filter combines Resources with the same GVK + N + NS into tuples, and then merges them
func (m Merge3) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
	// index the nodes by their identity
	matcher := m.Matcher
	if matcher == nil {
		matcher = &DefaultGVKNNMatcher{MergeOnPath: true}
	}
	handler := m.Handler
	if handler == nil {
		handler = &DefaultResourceHandler{}
	}

	tl := tuples{matcher: matcher}
	for i := range nodes {
		if err := tl.add(nodes[i]); err != nil {
			return nil, err
		}
	}

	// iterate over the inputs, merging as needed
	var output []*yaml.RNode
	for i := range tl.list {
		t := tl.list[i]
		strategy, err := handler.Handle(t.original, t.updated, t.dest)
		if err != nil {
			return nil, err
		}
		switch strategy {
		case Merge:
			node, err := t.merge()
			if err != nil {
				return nil, err
			}
			if node != nil {
				output = append(output, node)
			}
		case KeepDest:
			output = append(output, t.dest)
		case KeepUpdated:
			output = append(output, t.updated)
		case KeepOriginal:
			output = append(output, t.original)
		case Skip:
			// do nothing
		}
	}
	return output, nil
}

// tuples combines nodes with the same GVK + N + NS
type tuples struct {
	list []*tuple

	// matcher matches the resources for merge
	matcher ResourceMatcher
}

// DefaultGVKNNMatcher holds the default matching of resources implementation based on
// Group, Version, Kind, Name and Namespace of the resource
type DefaultGVKNNMatcher struct {
	// MergeOnPath will use the relative filepath as part of the merge key.
	// This may be necessary if the directory contains multiple copies of
	// the same resource, or resources patches.
	MergeOnPath bool
}

// IsSameResource returns true if metadata of node1 and metadata of node2 belongs to same logical resource
func (dm *DefaultGVKNNMatcher) IsSameResource(node1, node2 *yaml.RNode) bool {
	if node1 == nil || node2 == nil {
		return false
	}

	meta1, err := node1.GetMeta()
	if err != nil {
		return false
	}

	meta2, err := node2.GetMeta()
	if err != nil {
		return false
	}

	if meta1.Name != meta2.Name {
		return false
	}
	if meta1.Namespace != meta2.Namespace {
		return false
	}
	if meta1.APIVersion != meta2.APIVersion {
		return false
	}
	if meta1.Kind != meta2.Kind {
		return false
	}
	if dm.MergeOnPath {
		// directories may contain multiple copies of a resource with the same
		// name, namespace, apiVersion and kind -- e.g. kustomize patches, or
		// multiple environments
		// mergeOnPath configures the merge logic to use the path as part of the
		// resource key
		if meta1.Annotations[kioutil.PathAnnotation] != meta2.Annotations[kioutil.PathAnnotation] {
			return false
		}
	}
	return true
}

// add adds a node to the list, combining it with an existing matching Resource if found
func (ts *tuples) add(node *yaml.RNode) error {
	for i := range ts.list {
		t := ts.list[i]
		if ts.matcher.IsSameResource(addedNode(t), node) {
			return t.add(node)
		}
	}
	t := &tuple{}
	if err := t.add(node); err != nil {
		return err
	}
	ts.list = append(ts.list, t)
	return nil
}

// addedNode returns one on the existing added nodes in the tuple
func addedNode(t *tuple) *yaml.RNode {
	if t.updated != nil {
		return t.updated
	}
	if t.original != nil {
		return t.original
	}
	return t.dest
}

// tuple wraps an original, updated, and dest tuple for a given Resource
type tuple struct {
	original *yaml.RNode
	updated  *yaml.RNode
	dest     *yaml.RNode
}

// add sets the corresponding tuple field for the node
func (t *tuple) add(node *yaml.RNode) error {
	meta, err := node.GetMeta()
	if err != nil {
		return err
	}
	switch meta.Annotations[mergeSourceAnnotation] {
	case mergeSourceDest:
		if t.dest != nil {
			return duplicateError("local", meta.Annotations[kioutil.PathAnnotation])
		}
		t.dest = node
	case mergeSourceOriginal:
		if t.original != nil {
			return duplicateError("original upstream", meta.Annotations[kioutil.PathAnnotation])
		}
		t.original = node
	case mergeSourceUpdated:
		if t.updated != nil {
			return duplicateError("updated upstream", meta.Annotations[kioutil.PathAnnotation])
		}
		t.updated = node
	default:
		return fmt.Errorf("no source annotation for Resource")
	}
	return nil
}

// merge performs a 3-way merge on the tuple
func (t *tuple) merge() (*yaml.RNode, error) {
	return merge3.Merge(t.dest, t.original, t.updated)
}

// duplicateError returns duplicate resources error
func duplicateError(source, filePath string) error {
	return fmt.Errorf(`found duplicate %q resources in file %q, please refer to "update" documentation for the fix`, source, filePath)
}

// DefaultResourceHandler is the default implementation of the ResourceHandler
// interface. It uses the following rules:
// * Keep dest if resource only exists in dest.
// * Keep updated if resource added in updated.
// * Delete dest if updated has been deleted.
// * Don't add the resource back if removed from dest.
// * Otherwise merge.
type DefaultResourceHandler struct{}

func (*DefaultResourceHandler) Handle(original, updated, dest *yaml.RNode) (ResourceMergeStrategy, error) {
	switch {
	case original == nil && updated == nil && dest != nil:
		// added locally -- keep dest
		return KeepDest, nil
	case updated != nil && dest == nil:
		// added in the update -- add update
		return KeepUpdated, nil
	case original != nil && updated == nil:
		// deleted in the update
		return Skip, nil
	case original != nil && dest == nil:
		// deleted locally
		return Skip, nil
	default:
		// dest and updated are non-nil -- merge them
		return Merge, nil
	}
}
<|endoftext|>