<commit_before>package azurerm

import (
	"crypto/sha1"
	"encoding/base64"
	"encoding/hex"
	"fmt"
	"log"
	"strings"
	"sync"

	"github.com/Azure/azure-sdk-for-go/arm/resources/resources"
	"github.com/hashicorp/go-multierror"
	"github.com/hashicorp/terraform/helper/mutexkv"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/terraform"
)

// Provider returns a terraform.ResourceProvider.
func Provider() terraform.ResourceProvider {
	var p *schema.Provider
	p = &schema.Provider{
		Schema: map[string]*schema.Schema{
			"subscription_id": {
				Type:        schema.TypeString,
				Required:    true,
				DefaultFunc: schema.EnvDefaultFunc("ARM_SUBSCRIPTION_ID", ""),
			},

			"client_id": {
				Type:        schema.TypeString,
				Required:    true,
				DefaultFunc: schema.EnvDefaultFunc("ARM_CLIENT_ID", ""),
			},

			"client_secret": {
				Type:        schema.TypeString,
				Required:    true,
				DefaultFunc: schema.EnvDefaultFunc("ARM_CLIENT_SECRET", ""),
			},

			"tenant_id": {
				Type:        schema.TypeString,
				Required:    true,
				DefaultFunc: schema.EnvDefaultFunc("ARM_TENANT_ID", ""),
			},

			"environment": {
				Type:        schema.TypeString,
				Required:    true,
				DefaultFunc: schema.EnvDefaultFunc("ARM_ENVIRONMENT", "public"),
			},

			"skip_provider_registration": {
				Type:        schema.TypeBool,
				Optional:    true,
				DefaultFunc: schema.EnvDefaultFunc("ARM_SKIP_PROVIDER_REGISTRATION", false),
			},
		},

		DataSourcesMap: map[string]*schema.Resource{
			"azurerm_client_config":  dataSourceArmClientConfig(),
			"azurerm_resource_group": dataSourceArmResourceGroup(),
			"azurerm_public_ip":      dataSourceArmPublicIP(),
			"azurerm_managed_disk":   dataSourceArmManagedDisk(),
		},

		ResourcesMap: map[string]*schema.Resource{
			// These resources use the Azure ARM SDK
			"azurerm_application_insights": resourceArmApplicationInsights(),
			"azurerm_availability_set":     resourceArmAvailabilitySet(),
			"azurerm_cdn_endpoint":         resourceArmCdnEndpoint(),
			"azurerm_cdn_profile":          resourceArmCdnProfile(),
			"azurerm_container_registry":   resourceArmContainerRegistry(),
			"azurerm_container_service":    resourceArmContainerService(),
			"azurerm_cosmosdb_account":     resourceArmCosmosDBAccount(),

			"azurerm_dns_a_record":     resourceArmDnsARecord(),
			"azurerm_dns_aaaa_record":  resourceArmDnsAAAARecord(),
			"azurerm_dns_cname_record": resourceArmDnsCNameRecord(),
			"azurerm_dns_mx_record":    resourceArmDnsMxRecord(),
			"azurerm_dns_ns_record":    resourceArmDnsNsRecord(),
			"azurerm_dns_ptr_record":   resourceArmDnsPtrRecord(),
			"azurerm_dns_srv_record":   resourceArmDnsSrvRecord(),
			"azurerm_dns_txt_record":   resourceArmDnsTxtRecord(),
			"azurerm_dns_zone":         resourceArmDnsZone(),

			"azurerm_eventgrid_topic":             resourceArmEventGridTopic(),
			"azurerm_eventhub":                    resourceArmEventHub(),
			"azurerm_eventhub_authorization_rule": resourceArmEventHubAuthorizationRule(),
			"azurerm_eventhub_consumer_group":     resourceArmEventHubConsumerGroup(),
			"azurerm_eventhub_namespace":          resourceArmEventHubNamespace(),
			"azurerm_express_route_circuit":       resourceArmExpressRouteCircuit(),

			"azurerm_image":            resourceArmImage(),
			"azurerm_key_vault":        resourceArmKeyVault(),
			"azurerm_key_vault_secret": resourceArmKeyVaultSecret(),

			"azurerm_lb":                      resourceArmLoadBalancer(),
			"azurerm_lb_backend_address_pool": resourceArmLoadBalancerBackendAddressPool(),
			"azurerm_lb_nat_rule":             resourceArmLoadBalancerNatRule(),
			"azurerm_lb_nat_pool":             resourceArmLoadBalancerNatPool(),
			"azurerm_lb_probe":                resourceArmLoadBalancerProbe(),
			"azurerm_lb_rule":                 resourceArmLoadBalancerRule(),
			"azurerm_local_network_gateway":   resourceArmLocalNetworkGateway(),

			"azurerm_managed_disk":           resourceArmManagedDisk(),
			"azurerm_network_interface":      resourceArmNetworkInterface(),
			"azurerm_network_security_group": resourceArmNetworkSecurityGroup(),
			"azurerm_network_security_rule":  resourceArmNetworkSecurityRule(),
			"azurerm_public_ip":              resourceArmPublicIp(),

			"azurerm_redis_cache": resourceArmRedisCache(),
			"azurerm_route":       resourceArmRoute(),
			"azurerm_route_table": resourceArmRouteTable(),

			"azurerm_servicebus_namespace":    resourceArmServiceBusNamespace(),
			"azurerm_servicebus_queue":        resourceArmServiceBusQueue(),
			"azurerm_servicebus_subscription": resourceArmServiceBusSubscription(),
			"azurerm_servicebus_topic":        resourceArmServiceBusTopic(),
			"azurerm_sql_elasticpool":         resourceArmSqlElasticPool(),
			"azurerm_storage_account":         resourceArmStorageAccount(),
			"azurerm_storage_blob":            resourceArmStorageBlob(),
			"azurerm_storage_container":       resourceArmStorageContainer(),
			"azurerm_storage_share":           resourceArmStorageShare(),
			"azurerm_storage_queue":           resourceArmStorageQueue(),
			"azurerm_storage_table":           resourceArmStorageTable(),
			"azurerm_subnet":                  resourceArmSubnet(),

			"azurerm_template_deployment":       resourceArmTemplateDeployment(),
			"azurerm_traffic_manager_endpoint":  resourceArmTrafficManagerEndpoint(),
			"azurerm_traffic_manager_profile":   resourceArmTrafficManagerProfile(),
			"azurerm_virtual_machine_extension": resourceArmVirtualMachineExtensions(),
			"azurerm_virtual_machine":           resourceArmVirtualMachine(),
			"azurerm_virtual_machine_scale_set": resourceArmVirtualMachineScaleSet(),
			"azurerm_virtual_network":           resourceArmVirtualNetwork(),
			"azurerm_virtual_network_peering":   resourceArmVirtualNetworkPeering(),

			"azurerm_app_service_plan": resourceArmAppServicePlan(),

			// These resources use the Riviera SDK
			"azurerm_resource_group":    resourceArmResourceGroup(),
			"azurerm_search_service":    resourceArmSearchService(),
			"azurerm_sql_database":      resourceArmSqlDatabase(),
			"azurerm_sql_firewall_rule": resourceArmSqlFirewallRule(),
			"azurerm_sql_server":        resourceArmSqlServer(),
		},
	}

	p.ConfigureFunc = providerConfigure(p)

	return p
}

// Config is the configuration structure used to instantiate a
// new Azure management client.
type Config struct {
	ManagementURL string

	SubscriptionID           string
	ClientID                 string
	ClientSecret             string
	TenantID                 string
	Environment              string
	SkipProviderRegistration bool

	validateCredentialsOnce sync.Once
}

func (c *Config) validate() error {
	var err *multierror.Error

	if c.SubscriptionID == "" {
		err = multierror.Append(err, fmt.Errorf("Subscription ID must be configured for the AzureRM provider"))
	}
	if c.ClientID == "" {
		err = multierror.Append(err, fmt.Errorf("Client ID must be configured for the AzureRM provider"))
	}
	if c.ClientSecret == "" {
		err = multierror.Append(err, fmt.Errorf("Client Secret must be configured for the AzureRM provider"))
	}
	if c.TenantID == "" {
		err = multierror.Append(err, fmt.Errorf("Tenant ID must be configured for the AzureRM provider"))
	}
	if c.Environment == "" {
		err = multierror.Append(err, fmt.Errorf("Environment must be configured for the AzureRM provider"))
	}

	return err.ErrorOrNil()
}

func providerConfigure(p *schema.Provider) schema.ConfigureFunc {
	return func(d *schema.ResourceData) (interface{}, error) {
		config := &Config{
			SubscriptionID:           d.Get("subscription_id").(string),
			ClientID:                 d.Get("client_id").(string),
			ClientSecret:             d.Get("client_secret").(string),
			TenantID:                 d.Get("tenant_id").(string),
			Environment:              d.Get("environment").(string),
			SkipProviderRegistration: d.Get("skip_provider_registration").(bool),
		}

		if err := config.validate(); err != nil {
			return nil, err
		}

		client, err := config.getArmClient()
		if err != nil {
			return nil, err
		}

		client.StopContext = p.StopContext()

		// replaces the context between tests
		p.MetaReset = func() error {
			client.StopContext = p.StopContext()
			return nil
		}

		// List all the available providers and their registration state to avoid unnecessary
		// requests. This also lets us check if the provider credentials are correct.
		providerList, err := client.providers.List(nil, "")
		if err != nil {
			return nil, fmt.Errorf("Unable to list provider registration status, it is possible that this is due to invalid "+
				"credentials or the service principal does not have permission to use the Resource Manager API, Azure "+
				"error: %s", err)
		}

		if !config.SkipProviderRegistration {
			err = registerAzureResourceProvidersWithSubscription(*providerList.Value, client.providers)
			if err != nil {
				return nil, err
			}
		}

		return client, nil
	}
}

func registerProviderWithSubscription(providerName string, client resources.ProvidersClient) error {
	_, err := client.Register(providerName)
	if err != nil {
		return fmt.Errorf("Cannot register provider %s with Azure Resource Manager: %s.", providerName, err)
	}

	return nil
}

var providerRegistrationOnce sync.Once

func determineAzureResourceProvidersToRegister(providerList []resources.Provider) map[string]struct{} {
	providers := map[string]struct{}{
		"Microsoft.Cache":             struct{}{},
		"Microsoft.Cdn":               struct{}{},
		"Microsoft.Compute":           struct{}{},
		"Microsoft.ContainerRegistry": struct{}{},
		"Microsoft.ContainerService":  struct{}{},
		"Microsoft.DocumentDB":        struct{}{},
		"Microsoft.EventGrid":         struct{}{},
		"Microsoft.EventHub":          struct{}{},
		"Microsoft.KeyVault":          struct{}{},
		"microsoft.insights":          struct{}{},
		"Microsoft.Network":           struct{}{},
		"Microsoft.Resources":         struct{}{},
		"Microsoft.Search":            struct{}{},
		"Microsoft.ServiceBus":        struct{}{},
		"Microsoft.Sql":               struct{}{},
		"Microsoft.Storage":           struct{}{},
	}

	// filter out any providers already registered
	for _, p := range providerList {
		if _, ok := providers[*p.Namespace]; !ok {
			continue
		}

		if strings.ToLower(*p.RegistrationState) == "registered" {
			log.Printf("[DEBUG] Skipping provider registration for namespace %s\n", *p.Namespace)
			delete(providers, *p.Namespace)
		}
	}

	return providers
}

// registerAzureResourceProvidersWithSubscription uses the providers client to register
// all Azure resource providers which the Terraform provider may require (regardless of
// whether they are actually used by the configuration or not). It was confirmed by Microsoft
// that this is the approach their own internal tools also take.
func registerAzureResourceProvidersWithSubscription(providerList []resources.Provider, client resources.ProvidersClient) error {
	var err error
	providerRegistrationOnce.Do(func() {

		providers := determineAzureResourceProvidersToRegister(providerList)

		var wg sync.WaitGroup
		wg.Add(len(providers))
		for providerName := range providers {
			go func(p string) {
				defer wg.Done()
				log.Printf("[DEBUG] Registering provider with namespace %s\n", p)
				if innerErr := registerProviderWithSubscription(p, client); err != nil {
					err = innerErr
				}
			}(providerName)
		}
		wg.Wait()
	})

	return err
}

// armMutexKV is the instance of MutexKV for ARM resources
var armMutexKV = mutexkv.NewMutexKV()

// Resource group names can be capitalised, but we store them in lowercase.
// Use a custom diff function to avoid creation of new resources.
func resourceAzurermResourceGroupNameDiffSuppress(k, old, new string, d *schema.ResourceData) bool {
	return strings.ToLower(old) == strings.ToLower(new)
}

// ignoreCaseDiffSuppressFunc is a DiffSuppressFunc from helper/schema that is
// used to ignore any case-changes in a return value.
func ignoreCaseDiffSuppressFunc(k, old, new string, d *schema.ResourceData) bool {
	return strings.ToLower(old) == strings.ToLower(new)
}

// ignoreCaseStateFunc is a StateFunc from helper/schema that converts the
// supplied value to lower before saving to state for consistency.
func ignoreCaseStateFunc(val interface{}) string {
	return strings.ToLower(val.(string))
}

func userDataStateFunc(v interface{}) string {
	switch s := v.(type) {
	case string:
		s = base64Encode(s)
		hash := sha1.Sum([]byte(s))
		return hex.EncodeToString(hash[:])
	default:
		return ""
	}
}

// base64Encode encodes data if the input isn't already encoded using
// base64.StdEncoding.EncodeToString. If the input is already base64 encoded,
// return the original input unchanged.
func base64Encode(data string) string {
	// Check whether the data is already Base64 encoded; don't double-encode
	if isBase64Encoded(data) {
		return data
	}
	// data has not been encoded encode and return
	return base64.StdEncoding.EncodeToString([]byte(data))
}

func isBase64Encoded(data string) bool {
	_, err := base64.StdEncoding.DecodeString(data)
	return err == nil
}
<commit_msg>Removing the Riviera split in the Provider list<commit_after>package azurerm

import (
	"crypto/sha1"
	"encoding/base64"
	"encoding/hex"
	"fmt"
	"log"
	"strings"
	"sync"

	"github.com/Azure/azure-sdk-for-go/arm/resources/resources"
	"github.com/hashicorp/go-multierror"
	"github.com/hashicorp/terraform/helper/mutexkv"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/terraform"
)

// Provider returns a terraform.ResourceProvider.
func Provider() terraform.ResourceProvider {
	var p *schema.Provider
	p = &schema.Provider{
		Schema: map[string]*schema.Schema{
			"subscription_id": {
				Type:        schema.TypeString,
				Required:    true,
				DefaultFunc: schema.EnvDefaultFunc("ARM_SUBSCRIPTION_ID", ""),
			},

			"client_id": {
				Type:        schema.TypeString,
				Required:    true,
				DefaultFunc: schema.EnvDefaultFunc("ARM_CLIENT_ID", ""),
			},

			"client_secret": {
				Type:        schema.TypeString,
				Required:    true,
				DefaultFunc: schema.EnvDefaultFunc("ARM_CLIENT_SECRET", ""),
			},

			"tenant_id": {
				Type:        schema.TypeString,
				Required:    true,
				DefaultFunc: schema.EnvDefaultFunc("ARM_TENANT_ID", ""),
			},

			"environment": {
				Type:        schema.TypeString,
				Required:    true,
				DefaultFunc: schema.EnvDefaultFunc("ARM_ENVIRONMENT", "public"),
			},

			"skip_provider_registration": {
				Type:        schema.TypeBool,
				Optional:    true,
				DefaultFunc: schema.EnvDefaultFunc("ARM_SKIP_PROVIDER_REGISTRATION", false),
			},
		},

		DataSourcesMap: map[string]*schema.Resource{
			"azurerm_client_config":  dataSourceArmClientConfig(),
			"azurerm_resource_group": dataSourceArmResourceGroup(),
			"azurerm_public_ip":      dataSourceArmPublicIP(),
			"azurerm_managed_disk":   dataSourceArmManagedDisk(),
		},

		ResourcesMap: map[string]*schema.Resource{
			"azurerm_application_insights":        resourceArmApplicationInsights(),
			"azurerm_app_service_plan":            resourceArmAppServicePlan(),
			"azurerm_availability_set":            resourceArmAvailabilitySet(),
			"azurerm_cdn_endpoint":                resourceArmCdnEndpoint(),
			"azurerm_cdn_profile":                 resourceArmCdnProfile(),
			"azurerm_container_registry":          resourceArmContainerRegistry(),
			"azurerm_container_service":           resourceArmContainerService(),
			"azurerm_cosmosdb_account":            resourceArmCosmosDBAccount(),
			"azurerm_dns_a_record":                resourceArmDnsARecord(),
			"azurerm_dns_aaaa_record":             resourceArmDnsAAAARecord(),
			"azurerm_dns_cname_record":            resourceArmDnsCNameRecord(),
			"azurerm_dns_mx_record":               resourceArmDnsMxRecord(),
			"azurerm_dns_ns_record":               resourceArmDnsNsRecord(),
			"azurerm_dns_ptr_record":              resourceArmDnsPtrRecord(),
			"azurerm_dns_srv_record":              resourceArmDnsSrvRecord(),
			"azurerm_dns_txt_record":              resourceArmDnsTxtRecord(),
			"azurerm_dns_zone":                    resourceArmDnsZone(),
			"azurerm_eventgrid_topic":             resourceArmEventGridTopic(),
			"azurerm_eventhub":                    resourceArmEventHub(),
			"azurerm_eventhub_authorization_rule": resourceArmEventHubAuthorizationRule(),
			"azurerm_eventhub_consumer_group":     resourceArmEventHubConsumerGroup(),
			"azurerm_eventhub_namespace":          resourceArmEventHubNamespace(),
			"azurerm_express_route_circuit":       resourceArmExpressRouteCircuit(),
			"azurerm_image":                       resourceArmImage(),
			"azurerm_key_vault":                   resourceArmKeyVault(),
			"azurerm_key_vault_secret":            resourceArmKeyVaultSecret(),
			"azurerm_lb":                          resourceArmLoadBalancer(),
			"azurerm_lb_backend_address_pool":     resourceArmLoadBalancerBackendAddressPool(),
			"azurerm_lb_nat_rule":                 resourceArmLoadBalancerNatRule(),
			"azurerm_lb_nat_pool":                 resourceArmLoadBalancerNatPool(),
			"azurerm_lb_probe":                    resourceArmLoadBalancerProbe(),
			"azurerm_lb_rule":                     resourceArmLoadBalancerRule(),
			"azurerm_local_network_gateway":       resourceArmLocalNetworkGateway(),
			"azurerm_managed_disk":                resourceArmManagedDisk(),
			"azurerm_network_interface":           resourceArmNetworkInterface(),
			"azurerm_network_security_group":      resourceArmNetworkSecurityGroup(),
			"azurerm_network_security_rule":       resourceArmNetworkSecurityRule(),
			"azurerm_public_ip":                   resourceArmPublicIp(),
			"azurerm_redis_cache":                 resourceArmRedisCache(),
			"azurerm_resource_group":              resourceArmResourceGroup(),
			"azurerm_route":                       resourceArmRoute(),
			"azurerm_route_table":                 resourceArmRouteTable(),
			"azurerm_search_service":              resourceArmSearchService(),
			"azurerm_servicebus_namespace":        resourceArmServiceBusNamespace(),
			"azurerm_servicebus_queue":            resourceArmServiceBusQueue(),
			"azurerm_servicebus_subscription":     resourceArmServiceBusSubscription(),
			"azurerm_servicebus_topic":            resourceArmServiceBusTopic(),
			"azurerm_sql_database":                resourceArmSqlDatabase(),
			"azurerm_sql_elasticpool":             resourceArmSqlElasticPool(),
			"azurerm_sql_firewall_rule":           resourceArmSqlFirewallRule(),
			"azurerm_sql_server":                  resourceArmSqlServer(),
			"azurerm_storage_account":             resourceArmStorageAccount(),
			"azurerm_storage_blob":                resourceArmStorageBlob(),
			"azurerm_storage_container":           resourceArmStorageContainer(),
			"azurerm_storage_share":               resourceArmStorageShare(),
			"azurerm_storage_queue":               resourceArmStorageQueue(),
			"azurerm_storage_table":               resourceArmStorageTable(),
			"azurerm_subnet":                      resourceArmSubnet(),
			"azurerm_template_deployment":         resourceArmTemplateDeployment(),
			"azurerm_traffic_manager_endpoint":    resourceArmTrafficManagerEndpoint(),
			"azurerm_traffic_manager_profile":     resourceArmTrafficManagerProfile(),
			"azurerm_virtual_machine_extension":   resourceArmVirtualMachineExtensions(),
			"azurerm_virtual_machine":             resourceArmVirtualMachine(),
			"azurerm_virtual_machine_scale_set":   resourceArmVirtualMachineScaleSet(),
			"azurerm_virtual_network":             resourceArmVirtualNetwork(),
			"azurerm_virtual_network_peering":     resourceArmVirtualNetworkPeering(),
		},
	}

	p.ConfigureFunc = providerConfigure(p)

	return p
}

// Config is the configuration structure used to instantiate a
// new Azure management client.
type Config struct {
	ManagementURL string

	SubscriptionID           string
	ClientID                 string
	ClientSecret             string
	TenantID                 string
	Environment              string
	SkipProviderRegistration bool

	validateCredentialsOnce sync.Once
}

func (c *Config) validate() error {
	var err *multierror.Error

	if c.SubscriptionID == "" {
		err = multierror.Append(err, fmt.Errorf("Subscription ID must be configured for the AzureRM provider"))
	}
	if c.ClientID == "" {
		err = multierror.Append(err, fmt.Errorf("Client ID must be configured for the AzureRM provider"))
	}
	if c.ClientSecret == "" {
		err = multierror.Append(err, fmt.Errorf("Client Secret must be configured for the AzureRM provider"))
	}
	if c.TenantID == "" {
		err = multierror.Append(err, fmt.Errorf("Tenant ID must be configured for the AzureRM provider"))
	}
	if c.Environment == "" {
		err = multierror.Append(err, fmt.Errorf("Environment must be configured for the AzureRM provider"))
	}

	return err.ErrorOrNil()
}

func providerConfigure(p *schema.Provider) schema.ConfigureFunc {
	return func(d *schema.ResourceData) (interface{}, error) {
		config := &Config{
			SubscriptionID:           d.Get("subscription_id").(string),
			ClientID:                 d.Get("client_id").(string),
			ClientSecret:             d.Get("client_secret").(string),
			TenantID:                 d.Get("tenant_id").(string),
			Environment:              d.Get("environment").(string),
			SkipProviderRegistration: d.Get("skip_provider_registration").(bool),
		}

		if err := config.validate(); err != nil {
			return nil, err
		}

		client, err := config.getArmClient()
		if err != nil {
			return nil, err
		}

		client.StopContext = p.StopContext()

		// replaces the context between tests
		p.MetaReset = func() error {
			client.StopContext = p.StopContext()
			return nil
		}

		// List all the available providers and their registration state to avoid unnecessary
		// requests. This also lets us check if the provider credentials are correct.
		providerList, err := client.providers.List(nil, "")
		if err != nil {
			return nil, fmt.Errorf("Unable to list provider registration status, it is possible that this is due to invalid "+
				"credentials or the service principal does not have permission to use the Resource Manager API, Azure "+
				"error: %s", err)
		}

		if !config.SkipProviderRegistration {
			err = registerAzureResourceProvidersWithSubscription(*providerList.Value, client.providers)
			if err != nil {
				return nil, err
			}
		}

		return client, nil
	}
}

func registerProviderWithSubscription(providerName string, client resources.ProvidersClient) error {
	_, err := client.Register(providerName)
	if err != nil {
		return fmt.Errorf("Cannot register provider %s with Azure Resource Manager: %s.", providerName, err)
	}

	return nil
}

var providerRegistrationOnce sync.Once

func determineAzureResourceProvidersToRegister(providerList []resources.Provider) map[string]struct{} {
	providers := map[string]struct{}{
		"Microsoft.Cache":             struct{}{},
		"Microsoft.Cdn":               struct{}{},
		"Microsoft.Compute":           struct{}{},
		"Microsoft.ContainerRegistry": struct{}{},
		"Microsoft.ContainerService":  struct{}{},
		"Microsoft.DocumentDB":        struct{}{},
		"Microsoft.EventGrid":         struct{}{},
		"Microsoft.EventHub":          struct{}{},
		"Microsoft.KeyVault":          struct{}{},
		"microsoft.insights":          struct{}{},
		"Microsoft.Network":           struct{}{},
		"Microsoft.Resources":         struct{}{},
		"Microsoft.Search":            struct{}{},
		"Microsoft.ServiceBus":        struct{}{},
		"Microsoft.Sql":               struct{}{},
		"Microsoft.Storage":           struct{}{},
	}

	// filter out any providers already registered
	for _, p := range providerList {
		if _, ok := providers[*p.Namespace]; !ok {
			continue
		}

		if strings.ToLower(*p.RegistrationState) == "registered" {
			log.Printf("[DEBUG] Skipping provider registration for namespace %s\n", *p.Namespace)
			delete(providers, *p.Namespace)
		}
	}

	return providers
}

// registerAzureResourceProvidersWithSubscription uses the providers client to register
// all Azure resource providers which the Terraform provider may require (regardless of
// whether they are actually used by the configuration or not). It was confirmed by Microsoft
// that this is the approach their own internal tools also take.
func registerAzureResourceProvidersWithSubscription(providerList []resources.Provider, client resources.ProvidersClient) error {
	var err error
	providerRegistrationOnce.Do(func() {

		providers := determineAzureResourceProvidersToRegister(providerList)

		var wg sync.WaitGroup
		wg.Add(len(providers))
		for providerName := range providers {
			go func(p string) {
				defer wg.Done()
				log.Printf("[DEBUG] Registering provider with namespace %s\n", p)
				if innerErr := registerProviderWithSubscription(p, client); err != nil {
					err = innerErr
				}
			}(providerName)
		}
		wg.Wait()
	})

	return err
}

// armMutexKV is the instance of MutexKV for ARM resources
var armMutexKV = mutexkv.NewMutexKV()

// Resource group names can be capitalised, but we store them in lowercase.
// Use a custom diff function to avoid creation of new resources.
func resourceAzurermResourceGroupNameDiffSuppress(k, old, new string, d *schema.ResourceData) bool {
	return strings.ToLower(old) == strings.ToLower(new)
}

// ignoreCaseDiffSuppressFunc is a DiffSuppressFunc from helper/schema that is
// used to ignore any case-changes in a return value.
func ignoreCaseDiffSuppressFunc(k, old, new string, d *schema.ResourceData) bool {
	return strings.ToLower(old) == strings.ToLower(new)
}

// ignoreCaseStateFunc is a StateFunc from helper/schema that converts the
// supplied value to lower before saving to state for consistency.
func ignoreCaseStateFunc(val interface{}) string {
	return strings.ToLower(val.(string))
}

func userDataStateFunc(v interface{}) string {
	switch s := v.(type) {
	case string:
		s = base64Encode(s)
		hash := sha1.Sum([]byte(s))
		return hex.EncodeToString(hash[:])
	default:
		return ""
	}
}

// base64Encode encodes data if the input isn't already encoded using
// base64.StdEncoding.EncodeToString. If the input is already base64 encoded,
// return the original input unchanged.
func base64Encode(data string) string {
	// Check whether the data is already Base64 encoded; don't double-encode
	if isBase64Encoded(data) {
		return data
	}
	// data has not been encoded encode and return
	return base64.StdEncoding.EncodeToString([]byte(data))
}

func isBase64Encoded(data string) bool {
	_, err := base64.StdEncoding.DecodeString(data)
	return err == nil
}
<|endoftext|>


<commit_before>// Copyright ©2014 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package fd provides functions to approximate derivatives using finite differences.
package fd

import (
	"math"
	"runtime"
	"sync"

	"github.com/gonum/floats"
)

// A Point is a stencil location in a difference method.
type Point struct {
	Loc   float64
	Coeff float64
}

// Method is a specific finite difference method. Method specifies the stencil,
// that is, the function locations (relative to x) which will be used to estimate
// the derivative. It also specifies the order of derivative it estimates. Order = 1
// represents the derivative, Order = 2 represents the curvature, etc.
type Method struct {
	Stencil []Point
	Order   int     // The order of the difference method (first derivative, second derivative, etc.)
	Step    float64 // Default step size for the method.
}

// Settings is the settings structure for computing finite differences.
type Settings struct {
	OriginKnown bool    // Flag that the value at the origin x is known
	OriginValue float64 // Value at the origin (only used if OriginKnown is true)
	Concurrent  bool    // Should the function calls be executed concurrently
	Workers     int     // Maximum number of concurrent executions when evaluating concurrently
	Method      Method  // Finite difference method to use
}

// DefaultSettings is a basic set of settings for computing finite differences.
// Computes a central difference approximation for the first derivative
// of the function.
func DefaultSettings() *Settings {
	return &Settings{
		Method:  Central,
		Workers: runtime.GOMAXPROCS(0),
	}
}

// Derivative estimates the derivative of the function f at the given location.
// The order of derivative, sample locations, and other options are specified
// by settings.
func Derivative(f func(float64) float64, x float64, settings *Settings) float64 {
	if settings == nil {
		settings = DefaultSettings()
	}
	step := settings.Method.Step
	var deriv float64
	method := settings.Method
	if !settings.Concurrent {
		for _, pt := range method.Stencil {
			if settings.OriginKnown && pt.Loc == 0 {
				deriv += pt.Coeff * settings.OriginValue
				continue
			}
			deriv += pt.Coeff * f(x+step*pt.Loc)
		}
		return deriv / math.Pow(step, float64(method.Order))
	}

	wg := &sync.WaitGroup{}
	mux := &sync.Mutex{}
	for _, pt := range method.Stencil {
		if settings.OriginKnown && pt.Loc == 0 {
			mux.Lock()
			deriv += pt.Coeff * settings.OriginValue
			mux.Unlock()
			continue
		}
		wg.Add(1)
		go func(pt Point) {
			defer wg.Done()
			fofx := f(x + step*pt.Loc)
			mux.Lock()
			defer mux.Unlock()
			deriv += pt.Coeff * fofx
		}(pt)
	}
	wg.Wait()
	return deriv / math.Pow(step, float64(method.Order))
}

// Gradient estimates the gradient of the multivariate function f at the
// location x. The result is stored in-place into dst if dst is not nil,
// otherwise a new slice will be allocated and returned. Finite difference
// kernel and other options are specified by settings. If settings is nil,
// default settings will be used.
// Gradient panics if the length of dst and x is not equal.
func Gradient(dst []float64, f func([]float64) float64, x []float64, settings *Settings) []float64 {
	if dst == nil {
		dst = make([]float64, len(x))
	}
	if len(dst) != len(x) {
		panic("fd: slice length mismatch")
	}
	if settings == nil {
		settings = DefaultSettings()
	}
	step := settings.Method.Step
	if !settings.Concurrent {
		xcopy := make([]float64, len(x)) // So that x is not modified during the call
		copy(xcopy, x)
		for i := range xcopy {
			var deriv float64
			for _, pt := range settings.Method.Stencil {
				if settings.OriginKnown && pt.Loc == 0 {
					deriv += pt.Coeff * settings.OriginValue
					continue
				}
				xcopy[i] += pt.Loc * step
				deriv += pt.Coeff * f(xcopy)
				xcopy[i] = x[i]
			}
			dst[i] = deriv / math.Pow(step, float64(settings.Method.Order))
		}
		return dst
	}

	nWorkers := settings.Workers
	expect := len(settings.Method.Stencil) * len(x)
	if nWorkers > expect {
		nWorkers = expect
	}

	quit := make(chan struct{})
	defer close(quit)
	sendChan := make(chan fdrun, expect)
	ansChan := make(chan fdrun, expect)

	// Launch workers. Workers receive an index and a step, and compute the answer
	for i := 0; i < settings.Workers; i++ {
		go func(sendChan <-chan fdrun, ansChan chan<- fdrun, quit <-chan struct{}) {
			xcopy := make([]float64, len(x))
			copy(xcopy, x)
			for {
				select {
				case <-quit:
					return
				case run := <-sendChan:
					xcopy[run.idx] += run.pt.Loc * step
					run.result = f(xcopy)
					xcopy[run.idx] = x[run.idx]
					ansChan <- run
				}
			}
		}(sendChan, ansChan, quit)
	}

	// Launch the distributor. Distributor sends the cases to be computed
	go func(sendChan chan<- fdrun, ansChan chan<- fdrun) {
		for i := range x {
			for _, pt := range settings.Method.Stencil {
				if settings.OriginKnown && pt.Loc == 0 {
					// Answer already known. Send the answer on the answer channel
					ansChan <- fdrun{
						idx:    i,
						pt:     pt,
						result: settings.OriginValue,
					}
					continue
				}
				// Answer not known, send the answer to be computed
				sendChan <- fdrun{
					idx: i,
					pt:  pt,
				}
			}
		}
	}(sendChan, ansChan)

	for i := range dst {
		dst[i] = 0
	}
	// Read in all of the results
	for i := 0; i < expect; i++ {
		run := <-ansChan
		dst[run.idx] += run.pt.Coeff * run.result
	}
	floats.Scale(1/math.Pow(step, float64(settings.Method.Order)), dst)
	return dst
}

type fdrun struct {
	idx    int
	pt     Point
	result float64
}

// Forward represents a first-order forward difference.
var Forward = Method{
	Stencil: []Point{{Loc: 0, Coeff: -1}, {Loc: 1, Coeff: 1}},
	Order:   1,
	Step:    1e-6,
}

// Backward represents a first-order backward difference.
var Backward = Method{
	Stencil: []Point{{Loc: -1, Coeff: -1}, {Loc: 0, Coeff: 1}},
	Order:   1,
	Step:    1e-6,
}

// Central represents a first-order central difference.
var Central = Method{
	Stencil: []Point{{Loc: -1, Coeff: -0.5}, {Loc: 1, Coeff: 0.5}},
	Order:   1,
	Step:    1e-6,
}

// Central2nd represents a secord-order central difference.
var Central2nd = Method{
	Stencil: []Point{{Loc: -1, Coeff: 1}, {Loc: 0, Coeff: -2}, {Loc: 1, Coeff: 1}},
	Order:   2,
	Step:    1e-3,
}
<commit_msg>fd: update doc comment for Derivative<commit_after>// Copyright ©2014 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package fd provides functions to approximate derivatives using finite differences.
package fd

import (
	"math"
	"runtime"
	"sync"

	"github.com/gonum/floats"
)

// A Point is a stencil location in a difference method.
type Point struct {
	Loc   float64
	Coeff float64
}

// Method is a specific finite difference method. Method specifies the stencil,
// that is, the function locations (relative to x) which will be used to estimate
// the derivative. It also specifies the order of derivative it estimates. Order = 1
// represents the derivative, Order = 2 represents the curvature, etc.
type Method struct {
	Stencil []Point
	Order   int     // The order of the difference method (first derivative, second derivative, etc.)
	Step    float64 // Default step size for the method.
}

// Settings is the settings structure for computing finite differences.
type Settings struct {
	OriginKnown bool    // Flag that the value at the origin x is known
	OriginValue float64 // Value at the origin (only used if OriginKnown is true)
	Concurrent  bool    // Should the function calls be executed concurrently
	Workers     int     // Maximum number of concurrent executions when evaluating concurrently
	Method      Method  // Finite difference method to use
}

// DefaultSettings is a basic set of settings for computing finite differences.
// Computes a central difference approximation for the first derivative
// of the function.
func DefaultSettings() *Settings {
	return &Settings{
		Method:  Central,
		Workers: runtime.GOMAXPROCS(0),
	}
}

// Derivative estimates the derivative of the function f at the given location.
// The order of the derivative, sample locations, and other options are
// specified by settings. If settings is nil, default settings will be used.
func Derivative(f func(float64) float64, x float64, settings *Settings) float64 {
	if settings == nil {
		settings = DefaultSettings()
	}
	step := settings.Method.Step
	var deriv float64
	method := settings.Method
	if !settings.Concurrent {
		for _, pt := range method.Stencil {
			if settings.OriginKnown && pt.Loc == 0 {
				deriv += pt.Coeff * settings.OriginValue
				continue
			}
			deriv += pt.Coeff * f(x+step*pt.Loc)
		}
		return deriv / math.Pow(step, float64(method.Order))
	}

	wg := &sync.WaitGroup{}
	mux := &sync.Mutex{}
	for _, pt := range method.Stencil {
		if settings.OriginKnown && pt.Loc == 0 {
			mux.Lock()
			deriv += pt.Coeff * settings.OriginValue
			mux.Unlock()
			continue
		}
		wg.Add(1)
		go func(pt Point) {
			defer wg.Done()
			fofx := f(x + step*pt.Loc)
			mux.Lock()
			defer mux.Unlock()
			deriv += pt.Coeff * fofx
		}(pt)
	}
	wg.Wait()
	return deriv / math.Pow(step, float64(method.Order))
}

// Gradient estimates the gradient of the multivariate function f at the
// location x. The result is stored in-place into dst if dst is not nil,
// otherwise a new slice will be allocated and returned. Finite difference
// kernel and other options are specified by settings. If settings is nil,
// default settings will be used.
// Gradient panics if the length of dst and x is not equal.
func Gradient(dst []float64, f func([]float64) float64, x []float64, settings *Settings) []float64 {
	if dst == nil {
		dst = make([]float64, len(x))
	}
	if len(dst) != len(x) {
		panic("fd: slice length mismatch")
	}
	if settings == nil {
		settings = DefaultSettings()
	}
	step := settings.Method.Step
	if !settings.Concurrent {
		xcopy := make([]float64, len(x)) // So that x is not modified during the call
		copy(xcopy, x)
		for i := range xcopy {
			var deriv float64
			for _, pt := range settings.Method.Stencil {
				if settings.OriginKnown && pt.Loc == 0 {
					deriv += pt.Coeff * settings.OriginValue
					continue
				}
				xcopy[i] += pt.Loc * step
				deriv += pt.Coeff * f(xcopy)
				xcopy[i] = x[i]
			}
			dst[i] = deriv / math.Pow(step, float64(settings.Method.Order))
		}
		return dst
	}

	nWorkers := settings.Workers
	expect := len(settings.Method.Stencil) * len(x)
	if nWorkers > expect {
		nWorkers = expect
	}

	quit := make(chan struct{})
	defer close(quit)
	sendChan := make(chan fdrun, expect)
	ansChan := make(chan fdrun, expect)

	// Launch workers. Workers receive an index and a step, and compute the answer
	for i := 0; i < settings.Workers; i++ {
		go func(sendChan <-chan fdrun, ansChan chan<- fdrun, quit <-chan struct{}) {
			xcopy := make([]float64, len(x))
			copy(xcopy, x)
			for {
				select {
				case <-quit:
					return
				case run := <-sendChan:
					xcopy[run.idx] += run.pt.Loc * step
					run.result = f(xcopy)
					xcopy[run.idx] = x[run.idx]
					ansChan <- run
				}
			}
		}(sendChan, ansChan, quit)
	}

	// Launch the distributor. Distributor sends the cases to be computed
	go func(sendChan chan<- fdrun, ansChan chan<- fdrun) {
		for i := range x {
			for _, pt := range settings.Method.Stencil {
				if settings.OriginKnown && pt.Loc == 0 {
					// Answer already known. Send the answer on the answer channel
					ansChan <- fdrun{
						idx:    i,
						pt:     pt,
						result: settings.OriginValue,
					}
					continue
				}
				// Answer not known, send the answer to be computed
				sendChan <- fdrun{
					idx: i,
					pt:  pt,
				}
			}
		}
	}(sendChan, ansChan)

	for i := range dst {
		dst[i] = 0
	}
	// Read in all of the results
	for i := 0; i < expect; i++ {
		run := <-ansChan
		dst[run.idx] += run.pt.Coeff * run.result
	}
	floats.Scale(1/math.Pow(step, float64(settings.Method.Order)), dst)
	return dst
}

type fdrun struct {
	idx    int
	pt     Point
	result float64
}

// Forward represents a first-order forward difference.
var Forward = Method{
	Stencil: []Point{{Loc: 0, Coeff: -1}, {Loc: 1, Coeff: 1}},
	Order:   1,
	Step:    1e-6,
}

// Backward represents a first-order backward difference.
var Backward = Method{
	Stencil: []Point{{Loc: -1, Coeff: -1}, {Loc: 0, Coeff: 1}},
	Order:   1,
	Step:    1e-6,
}

// Central represents a first-order central difference.
var Central = Method{
	Stencil: []Point{{Loc: -1, Coeff: -0.5}, {Loc: 1, Coeff: 0.5}},
	Order:   1,
	Step:    1e-6,
}

// Central2nd represents a secord-order central difference.
var Central2nd = Method{
	Stencil: []Point{{Loc: -1, Coeff: 1}, {Loc: 0, Coeff: -2}, {Loc: 1, Coeff: 1}},
	Order:   2,
	Step:    1e-3,
}
<|endoftext|>


<commit_before>package skydb

import (
	"fmt"

	"github.com/oursky/skygear/skyerr"
)

// SortOrder denotes an the order of Records returned from a Query.
type SortOrder int

// A list of SordOrder, their meaning is self descriptive.
const (
	Ascending SortOrder = iota
	Descending
	Asc  = Ascending
	Desc = Descending
)

// Sort specifies the order of a collection of Records returned from a Query.
//
// Record order can be sorted w.r.t. a record field or a value returned
// from a predefined function.
type Sort struct {
	KeyPath string
	Func    Func
	Order   SortOrder
}

// Operator denotes how the result of a predicate is determined from
// its subpredicates or subexpressions.
//go:generate stringer -type=Operator
type Operator int

// A list of Operator.
const (
	And Operator = iota + 1
	Or
	Not
	Equal
	GreaterThan
	LessThan
	GreaterThanOrEqual
	LessThanOrEqual
	NotEqual
	Like
	ILike
	In
	Functional
)

// IsCompound checks whether the Operator is a compound operator, meaning the
// operator combine the results of other subpredicates.
func (op Operator) IsCompound() bool {
	switch op {
	default:
		return false
	case And, Or, Not:
		return true
	}
}

// IsBinary checks whether the Operator determines the result of a predicate
// by comparing two subexpressions.
func (op Operator) IsBinary() bool {
	switch op {
	default:
		return false
	case Equal, GreaterThan, LessThan, GreaterThanOrEqual, LessThanOrEqual, NotEqual, Like, ILike, In:
		return true
	}
}

// ExpressionType is the type of an Expression.
type ExpressionType int

// A list of ExpressionTypes.
const (
	Literal ExpressionType = iota + 1
	KeyPath
	Function
)

// An Expression represents value to be compared against.
type Expression struct {
	Type  ExpressionType
	Value interface{}
}

func (expr Expression) IsEmpty() bool {
	return expr.Type == 0 && expr.Value == nil
}

func (expr Expression) IsKeyPath() bool {
	return expr.Type == KeyPath
}

func (expr Expression) IsLiteralString() bool {
	if expr.Type != Literal {
		return false
	}

	_, ok := expr.Value.(string)
	return ok
}

func (expr Expression) IsLiteralArray() bool {
	if expr.Type != Literal {
		return false
	}

	_, ok := expr.Value.([]interface{})
	return ok
}

func (expr Expression) IsLiteralMap() bool {
	if expr.Type != Literal {
		return false
	}

	_, ok := expr.Value.(map[string]interface{})
	return ok
}

// Predicate is a representation of used in query for filtering records.
type Predicate struct {
	Operator Operator
	Children []interface{}
}

func (p Predicate) IsEmpty() bool {
	return p.Operator == 0 || p.Children == nil
}

// Validate returns an Error if a Predicate is invalid.
//
// If a Predicate is validated without error, nil is returned.
func (p Predicate) Validate() skyerr.Error {
	return p.validate(nil)
}

// validates is an internal version of the exported Validate() function.
//
// Additional information is passed as parameter to check the context
// in which the predicate is specified.
func (p Predicate) validate(parentPredicate *Predicate) skyerr.Error {
	if p.Operator.IsBinary() && len(p.Children) != 2 {
		return skyerr.NewErrorf(skyerr.InternalQueryInvalid,
			"binary predicate must have 2 operands, got %d", len(p.Children))
	}
	if p.Operator == Functional && len(p.Children) != 1 {
		return skyerr.NewErrorf(skyerr.InternalQueryInvalid,
			"functional predicate must have 1 operand, got %d", len(p.Children))
	}

	if p.Operator.IsCompound() {
		for _, child := range p.Children {
			predicate, ok := child.(Predicate)
			if !ok {
				return skyerr.NewError(skyerr.InternalQueryInvalid,
					"children of compound predicate must be a predicate")
			}

			if err := predicate.validate(&p); err != nil {
				return err
			}
		}
	} else {
		for _, child := range p.Children {
			_, ok := child.(Expression)
			if !ok {
				return skyerr.NewError(skyerr.InternalQueryInvalid,
					"children of simple predicate must be an expression")
			}
		}
	}

	switch p.Operator {
	case In:
		lhs := p.Children[0].(Expression)
		rhs := p.Children[1].(Expression)

		if lhs.IsKeyPath() == rhs.IsKeyPath() {
			return skyerr.NewError(skyerr.InternalQueryInvalid,
				`either one of the operands of "IN" must be key path`)
		}

		if rhs.IsKeyPath() && !lhs.IsLiteralString() {
			return skyerr.NewError(skyerr.InternalQueryInvalid,
				`left operand of "IN" must be a string if comparing with a keypath`)
		} else if lhs.IsKeyPath() && !rhs.IsLiteralArray() {
			return skyerr.NewError(skyerr.InternalQueryInvalid,
				`right operand of "IN" must be an array if comparing with a keypath`)
		}
	case Functional:
		expr := p.Children[0].(Expression)
		if expr.Type != Function {
			return skyerr.NewError(skyerr.InternalQueryInvalid,
				`functional predicate must contain functional expression`)
		}

		switch f := expr.Value.(type) {
		case UserRelationFunc:
			if f.RelationName != "_friend" && f.RelationName != "_follow" {
				return skyerr.NewErrorf(skyerr.NotSupported,
					`user relation predicate with "%d" relation is not supported`,
					f.RelationName)
			}
		case UserDiscoverFunc:
			if parentPredicate != nil {
				return skyerr.NewError(skyerr.NotSupported,
					`user discover predicate cannot be combined with other predicates`)
			}
		default:
			return skyerr.NewError(skyerr.NotSupported,
				`unsupported function for functional predicate`)
		}
	case Equal:
		lhs := p.Children[0].(Expression)
		rhs := p.Children[1].(Expression)

		if lhs.IsLiteralMap() {
			return skyerr.NewErrorf(skyerr.NotSupported,
				`equal comparison of map "%v" is not supported`,
				lhs.Value)
		} else if lhs.IsLiteralArray() {
			return skyerr.NewErrorf(skyerr.NotSupported,
				`equal comparison of array "%v" is not supported`,
				lhs.Value)
		} else if rhs.IsLiteralMap() {
			return skyerr.NewErrorf(skyerr.NotSupported,
				`equal comparison of map "%v" is not supported`,
				rhs.Value)
		} else if rhs.IsLiteralArray() {
			return skyerr.NewErrorf(skyerr.NotSupported,
				`equal comparison of array "%v" is not supported`,
				rhs.Value)
		}
	}
	return nil
}

// GetSubPredicates returns Predicate.Children as []Predicate.
//
// This method is only valid when Operator is either And, Or and Not. Caller
// is responsible to check for this preconditions. Otherwise the method
// will panic.
func (p Predicate) GetSubPredicates() (ps []Predicate) {
	for _, childPred := range p.Children {
		ps = append(ps, childPred.(Predicate))
	}
	return
}

// GetExpressions returns Predicate.Children as []Expression.
//
// This method is only valid when Operator is binary operator. Caller
// is responsible to check for this preconditions. Otherwise the method
// will panic.
func (p Predicate) GetExpressions() (ps []Expression) {
	for _, childPred := range p.Children {
		ps = append(ps, childPred.(Expression))
	}
	return
}

// Query specifies the type, predicate and sorting order of Database
// query.
// ReadableBy is a temp solution for ACL before a full predicate implemented.
type Query struct {
	Type         string
	Predicate    Predicate
	Sorts        []Sort
	ReadableBy   string
	ComputedKeys map[string]Expression
	DesiredKeys  []string
	GetCount     bool
	Limit        *uint64
	Offset       uint64
}

// Func is a marker interface to denote a type being a function in skydb.
//
// skydb's function receives zero or more arguments and returns a DataType
// as a result. Result data type is currently omitted in this interface since
// skygear doesn't use it internally yet. In the future it can be utilized to
// provide more extensive type checking at handler level.
type Func interface {
	Args() []interface{}
}

// DistanceFunc represents a function that calculates distance between
// a user supplied location and a Record's field
type DistanceFunc struct {
	Field    string
	Location Location
}

// Args implements the Func interface
func (f DistanceFunc) Args() []interface{} {
	return []interface{}{f.Field, f.Location}
}

// CountFunc represents a function that count number of rows matching
// a query
type CountFunc struct {
	OverallRecords bool
}

// Args implements the Func interface
func (f CountFunc) Args() []interface{} {
	return []interface{}{}
}

// UserRelationFunc represents a function that is used to evaulate
// whether a record satisfy certain user-based relation
type UserRelationFunc struct {
	KeyPath           string
	RelationName      string
	RelationDirection string
	User              string
}

// Args implements the Func interface
func (f UserRelationFunc) Args() []interface{} {
	return []interface{}{}
}

// UserDiscoverFunc searches for user reord having the specified user data, such
// as email addresses. Can only be used with user record.
type UserDiscoverFunc struct {
	Emails []string
}

// Args implements the Func interface
func (f UserDiscoverFunc) Args() []interface{} {
	panic("not supported")
}

// ArgsByName implements the Func interface
func (f UserDiscoverFunc) ArgsByName(name string) []interface{} {
	var data []string
	switch name {
	case "email":
		data = f.Emails
	default:
		panic(fmt.Errorf("not supported arg name %s", name))
	}

	args := make([]interface{}, len(data))
	for i, email := range data {
		args[i] = email
	}
	return args
}

// UserDataFunc is an expresssion to return an attribute of user info
// as email addresses. Can only be used with user record.
type UserDataFunc struct {
	DataName string
}

// Args implements the Func interface
func (f UserDataFunc) Args() []interface{} {
	return []interface{}{}
}
<commit_msg>Reduce complexity of predicate validation<commit_after>package skydb

import (
	"fmt"

	"github.com/oursky/skygear/skyerr"
)

// SortOrder denotes an the order of Records returned from a Query.
type SortOrder int

// A list of SordOrder, their meaning is self descriptive.
const (
	Ascending SortOrder = iota
	Descending
	Asc  = Ascending
	Desc = Descending
)

// Sort specifies the order of a collection of Records returned from a Query.
//
// Record order can be sorted w.r.t. a record field or a value returned
// from a predefined function.
type Sort struct {
	KeyPath string
	Func    Func
	Order   SortOrder
}

// Operator denotes how the result of a predicate is determined from
// its subpredicates or subexpressions.
//go:generate stringer -type=Operator
type Operator int

// A list of Operator.
const (
	And Operator = iota + 1
	Or
	Not
	Equal
	GreaterThan
	LessThan
	GreaterThanOrEqual
	LessThanOrEqual
	NotEqual
	Like
	ILike
	In
	Functional
)

// IsCompound checks whether the Operator is a compound operator, meaning the
// operator combine the results of other subpredicates.
func (op Operator) IsCompound() bool {
	switch op {
	default:
		return false
	case And, Or, Not:
		return true
	}
}

// IsBinary checks whether the Operator determines the result of a predicate
// by comparing two subexpressions.
func (op Operator) IsBinary() bool {
	switch op {
	default:
		return false
	case Equal, GreaterThan, LessThan, GreaterThanOrEqual, LessThanOrEqual, NotEqual, Like, ILike, In:
		return true
	}
}

// ExpressionType is the type of an Expression.
type ExpressionType int

// A list of ExpressionTypes.
const (
	Literal ExpressionType = iota + 1
	KeyPath
	Function
)

// An Expression represents value to be compared against.
type Expression struct {
	Type  ExpressionType
	Value interface{}
}

func (expr Expression) IsEmpty() bool {
	return expr.Type == 0 && expr.Value == nil
}

func (expr Expression) IsKeyPath() bool {
	return expr.Type == KeyPath
}

func (expr Expression) IsLiteralString() bool {
	if expr.Type != Literal {
		return false
	}

	_, ok := expr.Value.(string)
	return ok
}

func (expr Expression) IsLiteralArray() bool {
	if expr.Type != Literal {
		return false
	}

	_, ok := expr.Value.([]interface{})
	return ok
}

func (expr Expression) IsLiteralMap() bool {
	if expr.Type != Literal {
		return false
	}

	_, ok := expr.Value.(map[string]interface{})
	return ok
}

// Predicate is a representation of used in query for filtering records.
type Predicate struct {
	Operator Operator
	Children []interface{}
}

func (p Predicate) IsEmpty() bool {
	return p.Operator == 0 || p.Children == nil
}

// Validate returns an Error if a Predicate is invalid.
//
// If a Predicate is validated without error, nil is returned.
func (p Predicate) Validate() skyerr.Error {
	return p.validate(nil)
}

// validates is an internal version of the exported Validate() function.
//
// Additional information is passed as parameter to check the context
// in which the predicate is specified.
func (p Predicate) validate(parentPredicate *Predicate) skyerr.Error {
	if p.Operator.IsBinary() && len(p.Children) != 2 {
		return skyerr.NewErrorf(skyerr.InternalQueryInvalid,
			"binary predicate must have 2 operands, got %d", len(p.Children))
	}
	if p.Operator == Functional && len(p.Children) != 1 {
		return skyerr.NewErrorf(skyerr.InternalQueryInvalid,
			"functional predicate must have 1 operand, got %d", len(p.Children))
	}

	if p.Operator.IsCompound() {
		for _, child := range p.Children {
			predicate, ok := child.(Predicate)
			if !ok {
				return skyerr.NewError(skyerr.InternalQueryInvalid,
					"children of compound predicate must be a predicate")
			}

			if err := predicate.validate(&p); err != nil {
				return err
			}
		}
	} else {
		for _, child := range p.Children {
			_, ok := child.(Expression)
			if !ok {
				return skyerr.NewError(skyerr.InternalQueryInvalid,
					"children of simple predicate must be an expression")
			}
		}
	}

	switch p.Operator {
	case In:
		return p.validateInPredicate(parentPredicate)
	case Functional:
		return p.validateFunctionalPredicate(parentPredicate)
	case Equal:
		return p.validateEqualPredicate(parentPredicate)
	}
	return nil
}

func (p Predicate) validateInPredicate(parentPredicate *Predicate) skyerr.Error {
	lhs := p.Children[0].(Expression)
	rhs := p.Children[1].(Expression)

	if lhs.IsKeyPath() == rhs.IsKeyPath() {
		return skyerr.NewError(skyerr.InternalQueryInvalid,
			`either one of the operands of "IN" must be key path`)
	}

	if rhs.IsKeyPath() && !lhs.IsLiteralString() {
		return skyerr.NewError(skyerr.InternalQueryInvalid,
			`left operand of "IN" must be a string if comparing with a keypath`)
	} else if lhs.IsKeyPath() && !rhs.IsLiteralArray() {
		return skyerr.NewError(skyerr.InternalQueryInvalid,
			`right operand of "IN" must be an array if comparing with a keypath`)
	}
	return nil
}

func (p Predicate) validateFunctionalPredicate(parentPredicate *Predicate) skyerr.Error {
	expr := p.Children[0].(Expression)
	if expr.Type != Function {
		return skyerr.NewError(skyerr.InternalQueryInvalid,
			`functional predicate must contain functional expression`)
	}

	switch f := expr.Value.(type) {
	case UserRelationFunc:
		if f.RelationName != "_friend" && f.RelationName != "_follow" {
			return skyerr.NewErrorf(skyerr.NotSupported,
				`user relation predicate with "%d" relation is not supported`,
				f.RelationName)
		}
	case UserDiscoverFunc:
		if parentPredicate != nil {
			return skyerr.NewError(skyerr.NotSupported,
				`user discover predicate cannot be combined with other predicates`)
		}
	default:
		return skyerr.NewError(skyerr.NotSupported,
			`unsupported function for functional predicate`)
	}
	return nil
}

func (p Predicate) validateEqualPredicate(parentPredicate *Predicate) skyerr.Error {
	lhs := p.Children[0].(Expression)
	rhs := p.Children[1].(Expression)

	if lhs.IsLiteralMap() {
		return skyerr.NewErrorf(skyerr.NotSupported,
			`equal comparison of map "%v" is not supported`,
			lhs.Value)
	} else if lhs.IsLiteralArray() {
		return skyerr.NewErrorf(skyerr.NotSupported,
			`equal comparison of array "%v" is not supported`,
			lhs.Value)
	} else if rhs.IsLiteralMap() {
		return skyerr.NewErrorf(skyerr.NotSupported,
			`equal comparison of map "%v" is not supported`,
			rhs.Value)
	} else if rhs.IsLiteralArray() {
		return skyerr.NewErrorf(skyerr.NotSupported,
			`equal comparison of array "%v" is not supported`,
			rhs.Value)
	}
	return nil
}

// GetSubPredicates returns Predicate.Children as []Predicate.
//
// This method is only valid when Operator is either And, Or and Not. Caller
// is responsible to check for this preconditions. Otherwise the method
// will panic.
func (p Predicate) GetSubPredicates() (ps []Predicate) {
	for _, childPred := range p.Children {
		ps = append(ps, childPred.(Predicate))
	}
	return
}

// GetExpressions returns Predicate.Children as []Expression.
//
// This method is only valid when Operator is binary operator. Caller
// is responsible to check for this preconditions. Otherwise the method
// will panic.
func (p Predicate) GetExpressions() (ps []Expression) {
	for _, childPred := range p.Children {
		ps = append(ps, childPred.(Expression))
	}
	return
}

// Query specifies the type, predicate and sorting order of Database
// query.
// ReadableBy is a temp solution for ACL before a full predicate implemented.
type Query struct {
	Type         string
	Predicate    Predicate
	Sorts        []Sort
	ReadableBy   string
	ComputedKeys map[string]Expression
	DesiredKeys  []string
	GetCount     bool
	Limit        *uint64
	Offset       uint64
}

// Func is a marker interface to denote a type being a function in skydb.
//
// skydb's function receives zero or more arguments and returns a DataType
// as a result. Result data type is currently omitted in this interface since
// skygear doesn't use it internally yet. In the future it can be utilized to
// provide more extensive type checking at handler level.
type Func interface {
	Args() []interface{}
}

// DistanceFunc represents a function that calculates distance between
// a user supplied location and a Record's field
type DistanceFunc struct {
	Field    string
	Location Location
}

// Args implements the Func interface
func (f DistanceFunc) Args() []interface{} {
	return []interface{}{f.Field, f.Location}
}

// CountFunc represents a function that count number of rows matching
// a query
type CountFunc struct {
	OverallRecords bool
}

// Args implements the Func interface
func (f CountFunc) Args() []interface{} {
	return []interface{}{}
}

// UserRelationFunc represents a function that is used to evaulate
// whether a record satisfy certain user-based relation
type UserRelationFunc struct {
	KeyPath           string
	RelationName      string
	RelationDirection string
	User              string
}

// Args implements the Func interface
func (f UserRelationFunc) Args() []interface{} {
	return []interface{}{}
}

// UserDiscoverFunc searches for user reord having the specified user data, such
// as email addresses. Can only be used with user record.
type UserDiscoverFunc struct {
	Emails []string
}

// Args implements the Func interface
func (f UserDiscoverFunc) Args() []interface{} {
	panic("not supported")
}

// ArgsByName implements the Func interface
func (f UserDiscoverFunc) ArgsByName(name string) []interface{} {
	var data []string
	switch name {
	case "email":
		data = f.Emails
	default:
		panic(fmt.Errorf("not supported arg name %s", name))
	}

	args := make([]interface{}, len(data))
	for i, email := range data {
		args[i] = email
	}
	return args
}

// UserDataFunc is an expresssion to return an attribute of user info
// as email addresses. Can only be used with user record.
type UserDataFunc struct {
	DataName string
}

// Args implements the Func interface
func (f UserDataFunc) Args() []interface{} {
	return []interface{}{}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"strings"

	"gopkg.in/jackc/pgx.v2"

	"github.com/golang/groupcache"

	"github.com/LeKovr/dbrpc/workman"
	"github.com/LeKovr/go-base/logger"
)

// -----------------------------------------------------------------------------

// Processor gets value from cache and converts it into Result struct
func cacheFetcher(log *logger.Log, cacheGroup *groupcache.Group) workman.WorkerFunc {
	// https://github.com/capotej/groupcache-db-experiment
	return func(payload string) workman.Result {
		var data []byte
		log.Printf("asked for %s from groupcache", payload)
		err := cacheGroup.Get(nil, payload,
			groupcache.AllocatingByteSliceSink(&data))
		var res workman.Result
		if err != nil {
			res = workman.Result{Success: false, Error: err.Error()}
		} else {
			d := data[1:]
			if data[0] == 1 { // First byte stores success state (1: true, 0: false)
				raw := json.RawMessage(d)
				res = workman.Result{Success: true, Result: &raw}
			} else {
				res = workman.Result{Success: false, Error: string(d)}
			}
		}
		return res
	}
}

// -----------------------------------------------------------------------------

func dbFetcher(cfg *AplFlags, log *logger.Log, db *pgx.Conn) groupcache.GetterFunc {
	return func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
		log.Printf("asking for %s from dbserver", key)

		var args []string
		var data []byte

		//err := json.Unmarshal(key, &args)
		json.Unmarshal([]byte(key), &args)

		if args[0] == cfg.ArgDefFunc {

			q := fmt.Sprintf("select * from %s.%s($1)", cfg.Schema, args[0])

			rows, err := db.Query(q, args[1])
			if err != nil {
				return err
			}
			defer rows.Close()

			var res []ArgDef
			for rows.Next() {
				var a ArgDef
				err = rows.Scan(&a.ID, &a.Name, &a.Type, &a.Default, &a.AllowNull)
				if err != nil {
					log.Printf("Func err: (%+v)", rows)
					return err
				}
				res = append(res, a)
			}
			if rows.Err() != nil {
				return err
			}
			log.Printf("Func def: %s (%+v)", args[1], res)

			data, err = json.Marshal(res)
			if err != nil {
				return err
			}

		} else {
			q, vals := PrepareFuncSQL(cfg, args)
			log.Printf("Query: %s (%+v / %+v)", q, vals)
			rows, err := db.Query(q, vals...)
			if err != nil {
				return err
			}

			data, err = FetchSQLResult(rows, log)
			defer rows.Close()
			if err != nil {
				return err
			}

		}
		result := []byte{1} // status: success
		result = append(result, data...)

		dd := result[1:]
		log.Printf("Save data: %s", dd)
		dest.SetBytes([]byte(result))
		return nil
	}
}

// -----------------------------------------------------------------------------

// PrepareFuncSQL prepares sql query with args placeholders
func PrepareFuncSQL(cfg *AplFlags, args []string) (string, []interface{}) {
	mtd := args[0]
	argVals := args[1:]

	argValPrep := make([]interface{}, len(argVals))
	argIDs := make([]string, len(argVals))

	for i, v := range argVals {
		argIDs[i] = fmt.Sprintf("$%d", i+1)
		argValPrep[i] = v
	}

	argIDStr := strings.Join(argIDs, ",")

	q := fmt.Sprintf("select * from %s.%s(%s)", cfg.Schema, mtd, argIDStr)

	return q, argValPrep
}

// -----------------------------------------------------------------------------

// FetchSQLResult fetches sql result and marshalls it into json
func FetchSQLResult(rows *pgx.Rows, log *logger.Log) (data []byte, err error) {
	// http://stackoverflow.com/a/29164115
	columnDefs := rows.FieldDescriptions()
	//log.Debugf("=========== %+v", columnDefs)
	columns := []string{}
	types := []string{}
	for _, c := range columnDefs {
		columns = append(columns, c.Name)
		types = append(types, c.DataTypeName)
	}

	var tableData []map[string]interface{}
	for rows.Next() {
		values, err := rows.Values()
		if err != nil {
			log.Warningf("Value fetch error: %s", err.Error())
		}
		log.Debugf("Values: %+v", values)

		entry := make(map[string]interface{})
		for i, col := range columns {
			var v interface{}
			val := values[i]
			if types[i] == "json" || types[i] == "jsonb" {
				raw := fmt.Sprintf("%s", val)
				ref := json.RawMessage(raw)
				entry[col] = &ref
			} else {
				v = val
				entry[col] = v
			}
		}
		tableData = append(tableData, entry)

	}
	data, err = json.Marshal(tableData)
	return
}
<commit_msg>fix: printf format<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"strings"

	"gopkg.in/jackc/pgx.v2"

	"github.com/golang/groupcache"

	"github.com/LeKovr/dbrpc/workman"
	"github.com/LeKovr/go-base/logger"
)

// -----------------------------------------------------------------------------

// Processor gets value from cache and converts it into Result struct
func cacheFetcher(log *logger.Log, cacheGroup *groupcache.Group) workman.WorkerFunc {
	// https://github.com/capotej/groupcache-db-experiment
	return func(payload string) workman.Result {
		var data []byte
		log.Printf("asked for %s from groupcache", payload)
		err := cacheGroup.Get(nil, payload,
			groupcache.AllocatingByteSliceSink(&data))
		var res workman.Result
		if err != nil {
			res = workman.Result{Success: false, Error: err.Error()}
		} else {
			d := data[1:]
			if data[0] == 1 { // First byte stores success state (1: true, 0: false)
				raw := json.RawMessage(d)
				res = workman.Result{Success: true, Result: &raw}
			} else {
				res = workman.Result{Success: false, Error: string(d)}
			}
		}
		return res
	}
}

// -----------------------------------------------------------------------------

func dbFetcher(cfg *AplFlags, log *logger.Log, db *pgx.Conn) groupcache.GetterFunc {
	return func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
		log.Printf("asking for %s from dbserver", key)

		var args []string
		var data []byte

		//err := json.Unmarshal(key, &args)
		json.Unmarshal([]byte(key), &args)

		if args[0] == cfg.ArgDefFunc {

			q := fmt.Sprintf("select * from %s.%s($1)", cfg.Schema, args[0])

			rows, err := db.Query(q, args[1])
			if err != nil {
				return err
			}
			defer rows.Close()

			var res []ArgDef
			for rows.Next() {
				var a ArgDef
				err = rows.Scan(&a.ID, &a.Name, &a.Type, &a.Default, &a.AllowNull)
				if err != nil {
					log.Printf("Func err: (%+v)", rows)
					return err
				}
				res = append(res, a)
			}
			if rows.Err() != nil {
				return err
			}
			log.Printf("Func def: %s (%+v)", args[1], res)

			data, err = json.Marshal(res)
			if err != nil {
				return err
			}

		} else {
			q, vals := PrepareFuncSQL(cfg, args)
			log.Printf("Query: %s (%+v)", q, vals)
			rows, err := db.Query(q, vals...)
			if err != nil {
				return err
			}

			data, err = FetchSQLResult(rows, log)
			defer rows.Close()
			if err != nil {
				return err
			}

		}
		result := []byte{1} // status: success
		result = append(result, data...)

		dd := result[1:]
		log.Printf("Save data: %s", dd)
		dest.SetBytes([]byte(result))
		return nil
	}
}

// -----------------------------------------------------------------------------

// PrepareFuncSQL prepares sql query with args placeholders
func PrepareFuncSQL(cfg *AplFlags, args []string) (string, []interface{}) {
	mtd := args[0]
	argVals := args[1:]

	argValPrep := make([]interface{}, len(argVals))
	argIDs := make([]string, len(argVals))

	for i, v := range argVals {
		argIDs[i] = fmt.Sprintf("$%d", i+1)
		argValPrep[i] = v
	}

	argIDStr := strings.Join(argIDs, ",")

	q := fmt.Sprintf("select * from %s.%s(%s)", cfg.Schema, mtd, argIDStr)

	return q, argValPrep
}

// -----------------------------------------------------------------------------

// FetchSQLResult fetches sql result and marshalls it into json
func FetchSQLResult(rows *pgx.Rows, log *logger.Log) (data []byte, err error) {
	// http://stackoverflow.com/a/29164115
	columnDefs := rows.FieldDescriptions()
	//log.Debugf("=========== %+v", columnDefs)
	columns := []string{}
	types := []string{}
	for _, c := range columnDefs {
		columns = append(columns, c.Name)
		types = append(types, c.DataTypeName)
	}

	var tableData []map[string]interface{}
	for rows.Next() {
		values, err := rows.Values()
		if err != nil {
			log.Warningf("Value fetch error: %s", err.Error())
		}
		log.Debugf("Values: %+v", values)

		entry := make(map[string]interface{})
		for i, col := range columns {
			var v interface{}
			val := values[i]
			if types[i] == "json" || types[i] == "jsonb" {
				raw := fmt.Sprintf("%s", val)
				ref := json.RawMessage(raw)
				entry[col] = &ref
			} else {
				v = val
				entry[col] = v
			}
		}
		tableData = append(tableData, entry)

	}
	data, err = json.Marshal(tableData)
	return
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"net/url"

	html "code.google.com/p/go.net/html"
	atom "code.google.com/p/go.net/html/atom"
	"github.com/PuerkitoBio/goquery"
	log "github.com/cihub/seelog"
)

var (
	InvalidNode                 = errors.New("Node is not an anchor")
	InvalidNodeAttributeMissing = errors.New("Node does not contain the specified attribute")
)

type Fetcher interface {
	// Fetch returns the body of URL and
	// a slice of URLs found on that page.
	Fetch(url string) (body string, urls []string, err error)
}

type HttpFetcher struct{}

// fetch retrieves the page at the specified URL and extracts URLs
func (h *HttpFetcher) Fetch(url string) (string, []string, error) {

	doc, err := goquery.NewDocument(url)
	if err != nil {
		return "", nil, err
	}

	urls, err := h.extractLinks(doc)
	if err != nil {
		return "", urls, err
	}

	log.Debugf("URLs: %+v", urls)

	return "", urls, nil
}

// extractLinks from a document
func (h *HttpFetcher) extractLinks(doc *goquery.Document) ([]string, error) {

	// Blank slice to hold the links on this page
	urls := make([]string, 0)

	// Extract all 'a' elements from the document
	sel := doc.Find("a")
	if sel == nil {
		// Assume zero links on failure
		return nil, nil
	}

	// Range over links, and add them to the list if valid
	for i, n := range sel.Nodes {

		// Validate the node is a link, and extract the target URL
		href, err := h.extractValidHref(n)
		if err != nil || href == "" {
			continue
		}

		// Normalise the URL and add if valid
		if uri := h.normaliseUrl(doc.Url, href); uri != "" {
			log.Debugf("Node %v: %s", i, href)
			urls = append(urls, uri)
		}
	}

	return urls, nil
}

// validateLink is an anchor with a href, and extract normalised url
func (h *HttpFetcher) extractValidHref(n *html.Node) (string, error) {
	var href string

	// Confirm this node is an anchor element
	if n == nil || n.Type != html.ElementNode || n.DataAtom != atom.A {
		return href, InvalidNode
	}

	// Return the value of the href attr if it exists
	for _, a := range n.Attr {
		if a.Key == "href" && a.Val != "" {
			return a.Val, nil
		}
	}

	return "", InvalidNodeAttributeMissing
}

// normaliseUrl converts relative URLs to absolute URLs
func (h *HttpFetcher) normaliseUrl(parent *url.URL, urlString string) string {

	// Parse the string into a url.URL
	uri, err := url.Parse(urlString)
	if err != nil {
		log.Debugf("Failed to parse URL: %s", urlString)
		return ""
	}

	// Resolve references to get an absolute URL
	abs := parent.ResolveReference(uri).String()
	log.Debugf("Resolved: %s", abs)

	return abs
}
<commit_msg>Fix godoc typo<commit_after>package main

import (
	"errors"
	"net/url"

	html "code.google.com/p/go.net/html"
	atom "code.google.com/p/go.net/html/atom"
	"github.com/PuerkitoBio/goquery"
	log "github.com/cihub/seelog"
)

var (
	InvalidNode                 = errors.New("Node is not an anchor")
	InvalidNodeAttributeMissing = errors.New("Node does not contain the specified attribute")
)

type Fetcher interface {
	// Fetch returns the body of URL and
	// a slice of URLs found on that page.
	Fetch(url string) (body string, urls []string, err error)
}

type HttpFetcher struct{}

// Fetch retrieves the page at the specified URL and extracts URLs
func (h *HttpFetcher) Fetch(url string) (string, []string, error) {

	doc, err := goquery.NewDocument(url)
	if err != nil {
		return "", nil, err
	}

	urls, err := h.extractLinks(doc)
	if err != nil {
		return "", urls, err
	}

	log.Debugf("URLs: %+v", urls)

	return "", urls, nil
}

// extractLinks from a document
func (h *HttpFetcher) extractLinks(doc *goquery.Document) ([]string, error) {

	// Blank slice to hold the links on this page
	urls := make([]string, 0)

	// Extract all 'a' elements from the document
	sel := doc.Find("a")
	if sel == nil {
		// Assume zero links on failure
		return nil, nil
	}

	// Range over links, and add them to the list if valid
	for i, n := range sel.Nodes {

		// Validate the node is a link, and extract the target URL
		href, err := h.extractValidHref(n)
		if err != nil || href == "" {
			continue
		}

		// Normalise the URL and add if valid
		if uri := h.normaliseUrl(doc.Url, href); uri != "" {
			log.Debugf("Node %v: %s", i, href)
			urls = append(urls, uri)
		}
	}

	return urls, nil
}

// validateLink is an anchor with a href, and extract normalised url
func (h *HttpFetcher) extractValidHref(n *html.Node) (string, error) {
	var href string

	// Confirm this node is an anchor element
	if n == nil || n.Type != html.ElementNode || n.DataAtom != atom.A {
		return href, InvalidNode
	}

	// Return the value of the href attr if it exists
	for _, a := range n.Attr {
		if a.Key == "href" && a.Val != "" {
			return a.Val, nil
		}
	}

	return "", InvalidNodeAttributeMissing
}

// normaliseUrl converts relative URLs to absolute URLs
func (h *HttpFetcher) normaliseUrl(parent *url.URL, urlString string) string {

	// Parse the string into a url.URL
	uri, err := url.Parse(urlString)
	if err != nil {
		log.Debugf("Failed to parse URL: %s", urlString)
		return ""
	}

	// Resolve references to get an absolute URL
	abs := parent.ResolveReference(uri).String()
	log.Debugf("Resolved: %s", abs)

	return abs
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2011 Matt Jibson <matt.jibson@gmail.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

// Package fft provides forward and inverse fast Fourier transform functions.
package fft

import (
	"math"
	"os"
	"sync"
)

var factors_lock sync.RWMutex

// radix-2 factors
var factors = map[int][]complex128{}

// bluestein factors
var n2_factors = map[int][]complex128{}
var n2_inv_factors = map[int][]complex128{}

// Ensures the complex multiplication factors exist for an input array of length input_len.
func ensureFactors(input_len int) {
	var cos, sin float64

	factors_lock.Lock()

	for i := 4; i <= input_len; i <<= 1 {
		if factors[i] == nil {
			factors[i] = make([]complex128, i)
			for n := 0; n < i; n++ {
				if n == 0 {
					sin, cos = 0, 1
				} else if n*4 == i {
					sin, cos = -1, 0
				} else {
					sin, cos = math.Sincos(-2 * math.Pi / float64(i) * float64(n))
				}
				factors[i][n] = complex(cos, sin)
			}
		}
	}

	if n2_factors[input_len] == nil {
		n2_factors[input_len] = make([]complex128, input_len)
		n2_inv_factors[input_len] = make([]complex128, input_len)

		for i := 0; i < input_len; i++ {
			if i == 0 {
				sin, cos = 0, 1
			} else {
				sin, cos = math.Sincos(math.Pi / float64(input_len) * float64(i*i))
			}
			n2_factors[input_len][i] = complex(cos, sin)
			n2_inv_factors[input_len][i] = complex(cos, -sin)
		}
	}

	factors_lock.Unlock()
}

// FFTReal returns the forward FFT of the real-valued slice.
func FFTReal(x []float64) []complex128 {
	return FFT(toComplex(x))
}

// IFFTReal returns the inverse FFT of the real-valued slice.
func IFFTReal(x []float64) []complex128 {
	return IFFT(toComplex(x))
}

// toComplex returns the complex equivalent of the real-valued slice.
func toComplex(x []float64) []complex128 {
	y := make([]complex128, len(x))
	for n, v := range x {
		y[n] = complex(v, 0)
	}
	return y
}

// IFFT returns the inverse FFT of the complex-valued slice.
func IFFT(x []complex128) []complex128 {
	lx := len(x)
	r := make([]complex128, lx)

	// Reverse inputs, which is calculated with modulo N, hence x[0] as an outlier
	r[0] = x[0]
	for i := 1; i < lx; i++ {
		r[i] = x[lx-i]
	}

	r = FFT(r)

	N := complex(float64(lx), 0)
	for n, _ := range r {
		r[n] /= N
	}
	return r
}

// Convolve returns the convolution of x * y.
func Convolve(x, y []complex128) ([]complex128, os.Error) {
	if len(x) != len(y) {
		return []complex128{}, os.NewError("fft: input arrays are not of equal length")
	}

	fft_x := FFT(x)
	fft_y := FFT(y)

	r := make([]complex128, len(x))
	for i := 0; i < len(r); i++ {
		r[i] = fft_x[i] * fft_y[i]
	}

	return IFFT(r), nil
}

// FFT returns the forward FFT of the complex-valued slice.
func FFT(x []complex128) []complex128 {
	lx := len(x)

	// todo: non-hack handling length <= 1 cases
	if lx <= 1 {
		r := make([]complex128, lx)
		copy(r, x)
		return r
	}

	if isPowerOf2(lx) {
		return radix2FFT(x)
	}

	return bluesteinFFT(x)
}

// radix2FFT returns the FFT calculated using the radix-2 DIT Cooley-Tukey algorithm.
func radix2FFT(x []complex128) []complex128 {
	lx := len(x)
	ensureFactors(lx)

	lx_2 := lx / 2
	r := make([]complex128, lx) // result
	t := make([]complex128, lx) // temp
	copy(r, x)

	// split into even and odd parts for each stage
	for block_sz := lx; block_sz > 1; block_sz >>= 1 {
		i := 0
		bs_2 := block_sz / 2
		for block := 0; block < lx/block_sz; block++ {
			for n := 0; n < bs_2; n++ {
				bn := block_sz*block + n
				t[bn] = r[i]
				i++
				t[bn+bs_2] = r[i]
				i++
			}
		}
		copy(r, t)
	}

	for stage := 2; stage <= lx; stage <<= 1 {
		if stage == 2 { // 2-point transforms
			for n := 0; n < lx_2; n++ {
				t[n*2] = r[n*2] + r[n*2+1]
				t[n*2+1] = r[n*2] - r[n*2+1]
			}
		} else { // >2-point transforms
			blocks := lx / stage
			s_2 := stage / 2

			for n := 0; n < blocks; n++ {
				nb := n * stage
				for j := 0; j < s_2; j++ {
					w_n := r[j+nb+s_2] * factors[stage][j]
					t[j+nb] = r[j+nb] + w_n
					t[j+nb+s_2] = r[j+nb] - w_n
				}
			}
		}

		copy(r, t)
	}

	return r
}

// bluesteinFFT returns the FFT calculated using the Bluestein algorithm.
func bluesteinFFT(x []complex128) []complex128 {
	lx := len(x)
	a := zeroPad(x, nextPowerOf2(lx*2-1))
	la := len(a)
	ensureFactors(lx)

	for n, v := range x {
		a[n] = v * n2_inv_factors[lx][n]
	}

	b := make([]complex128, la)
	for i := 0; i < lx; i++ {
		b[i] = n2_factors[lx][i]

		if i != 0 {
			b[la-i] = n2_factors[lx][i]
		}
	}

	r, _ := Convolve(a, b)

	for i := 0; i < lx; i++ {
		r[i] *= n2_inv_factors[lx][i]
	}

	return r[:lx]
}

// isPowerOf2 returns true if x is a power of 2, else false.
func isPowerOf2(x int) bool {
	return x&(x-1) == 0
}

// nextPowerOf2 returns the next power of 2 >= x.
func nextPowerOf2(x int) int {
	if isPowerOf2(x) {
		return x
	}

	return int(math.Pow(2, math.Ceil(math.Log2(float64(x)))))
}

// zeroPad returns x with zeros appended to the end to the specified length.
// If len(x) == length, x is returned.
func zeroPad(x []complex128, length int) []complex128 {
	if len(x) == length {
		return x
	}

	r := make([]complex128, length)
	copy(r, x)
	return r
}

// zeroPad2 returns zeroPad of x, with the length as next power of 2 >= len(x).
func zeroPad2(x []complex128) []complex128 {
	return zeroPad(x, nextPowerOf2(len(x)))
}

// toComplex2 returns the complex equivalent of the real-valued matrix.
func toComplex2(x [][]float64) [][]complex128 {
	y := make([][]complex128, len(x))
	for n, v := range x {
		y[n] = toComplex(v)
	}
	return y
}

// FFT2Real returns the 2-dimensional, forward FFT of the real-valued matrix.
func FFT2Real(x [][]float64) ([][]complex128, os.Error) {
	return FFT2(toComplex2(x))
}

// FFT2 returns the 2-dimensional, forward FFT of the complex-valued matrix.
func FFT2(x [][]complex128) ([][]complex128, os.Error) {
	return computeFFT2(x, FFT)
}

// IFFT2Real returns the 2-dimensional, inverse FFT of the real-valued matrix.
func IFFT2Real(x [][]float64) ([][]complex128, os.Error) {
	return IFFT2(toComplex2(x))
}

// IFFT2 returns the 2-dimensional, inverse FFT of the complex-valued matrix.
func IFFT2(x [][]complex128) ([][]complex128, os.Error) {
	return computeFFT2(x, IFFT)
}

func computeFFT2(x [][]complex128, fftFunc func([]complex128) ([]complex128)) ([][]complex128, os.Error) {
	rows := len(x)
	if rows == 0 {
		return nil, os.NewError("fft: empty input array")
	}

	cols := len(x[0])
	r := make([][]complex128, rows)
	for i := 0; i < rows; i++ {
		if len(x[i]) != cols {
			return nil, os.NewError("fft: input matrix must have identical row lengths")
		}
		r[i] = make([]complex128, cols)
	}

	for i := 0; i < cols; i++ {
		t := make([]complex128, rows)
		for j := 0; j < rows; j++ {
			t[j] = x[j][i]
		}

		for n, v := range fftFunc(t) {
			r[n][i] = v
		}
	}

	for n, v := range r {
		r[n] = fftFunc(v)
	}

	return r, nil
}
<commit_msg>Gofmt<commit_after>/*
 * Copyright (c) 2011 Matt Jibson <matt.jibson@gmail.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

// Package fft provides forward and inverse fast Fourier transform functions.
package fft

import (
	"math"
	"os"
	"sync"
)

var factors_lock sync.RWMutex

// radix-2 factors
var factors = map[int][]complex128{}

// bluestein factors
var n2_factors = map[int][]complex128{}
var n2_inv_factors = map[int][]complex128{}

// Ensures the complex multiplication factors exist for an input array of length input_len.
func ensureFactors(input_len int) {
	var cos, sin float64

	factors_lock.Lock()

	for i := 4; i <= input_len; i <<= 1 {
		if factors[i] == nil {
			factors[i] = make([]complex128, i)
			for n := 0; n < i; n++ {
				if n == 0 {
					sin, cos = 0, 1
				} else if n*4 == i {
					sin, cos = -1, 0
				} else {
					sin, cos = math.Sincos(-2 * math.Pi / float64(i) * float64(n))
				}
				factors[i][n] = complex(cos, sin)
			}
		}
	}

	if n2_factors[input_len] == nil {
		n2_factors[input_len] = make([]complex128, input_len)
		n2_inv_factors[input_len] = make([]complex128, input_len)

		for i := 0; i < input_len; i++ {
			if i == 0 {
				sin, cos = 0, 1
			} else {
				sin, cos = math.Sincos(math.Pi / float64(input_len) * float64(i*i))
			}
			n2_factors[input_len][i] = complex(cos, sin)
			n2_inv_factors[input_len][i] = complex(cos, -sin)
		}
	}

	factors_lock.Unlock()
}

// FFTReal returns the forward FFT of the real-valued slice.
func FFTReal(x []float64) []complex128 {
	return FFT(toComplex(x))
}

// IFFTReal returns the inverse FFT of the real-valued slice.
func IFFTReal(x []float64) []complex128 {
	return IFFT(toComplex(x))
}

// toComplex returns the complex equivalent of the real-valued slice.
func toComplex(x []float64) []complex128 {
	y := make([]complex128, len(x))
	for n, v := range x {
		y[n] = complex(v, 0)
	}
	return y
}

// IFFT returns the inverse FFT of the complex-valued slice.
func IFFT(x []complex128) []complex128 {
	lx := len(x)
	r := make([]complex128, lx)

	// Reverse inputs, which is calculated with modulo N, hence x[0] as an outlier
	r[0] = x[0]
	for i := 1; i < lx; i++ {
		r[i] = x[lx-i]
	}

	r = FFT(r)

	N := complex(float64(lx), 0)
	for n, _ := range r {
		r[n] /= N
	}
	return r
}

// Convolve returns the convolution of x * y.
func Convolve(x, y []complex128) ([]complex128, os.Error) {
	if len(x) != len(y) {
		return []complex128{}, os.NewError("fft: input arrays are not of equal length")
	}

	fft_x := FFT(x)
	fft_y := FFT(y)

	r := make([]complex128, len(x))
	for i := 0; i < len(r); i++ {
		r[i] = fft_x[i] * fft_y[i]
	}

	return IFFT(r), nil
}

// FFT returns the forward FFT of the complex-valued slice.
func FFT(x []complex128) []complex128 {
	lx := len(x)

	// todo: non-hack handling length <= 1 cases
	if lx <= 1 {
		r := make([]complex128, lx)
		copy(r, x)
		return r
	}

	if isPowerOf2(lx) {
		return radix2FFT(x)
	}

	return bluesteinFFT(x)
}

// radix2FFT returns the FFT calculated using the radix-2 DIT Cooley-Tukey algorithm.
func radix2FFT(x []complex128) []complex128 {
	lx := len(x)
	ensureFactors(lx)

	lx_2 := lx / 2
	r := make([]complex128, lx) // result
	t := make([]complex128, lx) // temp
	copy(r, x)

	// split into even and odd parts for each stage
	for block_sz := lx; block_sz > 1; block_sz >>= 1 {
		i := 0
		bs_2 := block_sz / 2
		for block := 0; block < lx/block_sz; block++ {
			for n := 0; n < bs_2; n++ {
				bn := block_sz*block + n
				t[bn] = r[i]
				i++
				t[bn+bs_2] = r[i]
				i++
			}
		}
		copy(r, t)
	}

	for stage := 2; stage <= lx; stage <<= 1 {
		if stage == 2 { // 2-point transforms
			for n := 0; n < lx_2; n++ {
				t[n*2] = r[n*2] + r[n*2+1]
				t[n*2+1] = r[n*2] - r[n*2+1]
			}
		} else { // >2-point transforms
			blocks := lx / stage
			s_2 := stage / 2

			for n := 0; n < blocks; n++ {
				nb := n * stage
				for j := 0; j < s_2; j++ {
					w_n := r[j+nb+s_2] * factors[stage][j]
					t[j+nb] = r[j+nb] + w_n
					t[j+nb+s_2] = r[j+nb] - w_n
				}
			}
		}

		copy(r, t)
	}

	return r
}

// bluesteinFFT returns the FFT calculated using the Bluestein algorithm.
func bluesteinFFT(x []complex128) []complex128 {
	lx := len(x)
	a := zeroPad(x, nextPowerOf2(lx*2-1))
	la := len(a)
	ensureFactors(lx)

	for n, v := range x {
		a[n] = v * n2_inv_factors[lx][n]
	}

	b := make([]complex128, la)
	for i := 0; i < lx; i++ {
		b[i] = n2_factors[lx][i]

		if i != 0 {
			b[la-i] = n2_factors[lx][i]
		}
	}

	r, _ := Convolve(a, b)

	for i := 0; i < lx; i++ {
		r[i] *= n2_inv_factors[lx][i]
	}

	return r[:lx]
}

// isPowerOf2 returns true if x is a power of 2, else false.
func isPowerOf2(x int) bool {
	return x&(x-1) == 0
}

// nextPowerOf2 returns the next power of 2 >= x.
func nextPowerOf2(x int) int {
	if isPowerOf2(x) {
		return x
	}

	return int(math.Pow(2, math.Ceil(math.Log2(float64(x)))))
}

// zeroPad returns x with zeros appended to the end to the specified length.
// If len(x) == length, x is returned.
func zeroPad(x []complex128, length int) []complex128 {
	if len(x) == length {
		return x
	}

	r := make([]complex128, length)
	copy(r, x)
	return r
}

// zeroPad2 returns zeroPad of x, with the length as next power of 2 >= len(x).
func zeroPad2(x []complex128) []complex128 {
	return zeroPad(x, nextPowerOf2(len(x)))
}

// toComplex2 returns the complex equivalent of the real-valued matrix.
func toComplex2(x [][]float64) [][]complex128 {
	y := make([][]complex128, len(x))
	for n, v := range x {
		y[n] = toComplex(v)
	}
	return y
}

// FFT2Real returns the 2-dimensional, forward FFT of the real-valued matrix.
func FFT2Real(x [][]float64) ([][]complex128, os.Error) {
	return FFT2(toComplex2(x))
}

// FFT2 returns the 2-dimensional, forward FFT of the complex-valued matrix.
func FFT2(x [][]complex128) ([][]complex128, os.Error) {
	return computeFFT2(x, FFT)
}

// IFFT2Real returns the 2-dimensional, inverse FFT of the real-valued matrix.
func IFFT2Real(x [][]float64) ([][]complex128, os.Error) {
	return IFFT2(toComplex2(x))
}

// IFFT2 returns the 2-dimensional, inverse FFT of the complex-valued matrix.
func IFFT2(x [][]complex128) ([][]complex128, os.Error) {
	return computeFFT2(x, IFFT)
}

func computeFFT2(x [][]complex128, fftFunc func([]complex128) []complex128) ([][]complex128, os.Error) {
	rows := len(x)
	if rows == 0 {
		return nil, os.NewError("fft: empty input array")
	}

	cols := len(x[0])
	r := make([][]complex128, rows)
	for i := 0; i < rows; i++ {
		if len(x[i]) != cols {
			return nil, os.NewError("fft: input matrix must have identical row lengths")
		}
		r[i] = make([]complex128, cols)
	}

	for i := 0; i < cols; i++ {
		t := make([]complex128, rows)
		for j := 0; j < rows; j++ {
			t[j] = x[j][i]
		}

		for n, v := range fftFunc(t) {
			r[n][i] = v
		}
	}

	for n, v := range r {
		r[n] = fftFunc(v)
	}

	return r, nil
}
<|endoftext|>


<commit_before>// Get a value from a slice and check whether a value exists in a slice.
//
// If the index is out-of-bounds, return the default value.
//
package slice

import (
	"reflect"

	"github.com/xgfone/go-tools/compare"
)

func setValue(out interface{}, slice interface{}, index int, _default interface{}, yes bool) bool {
	_out := reflect.Indirect(reflect.ValueOf(out))
	if !_out.CanSet() {
		return false
	}

	_slice := reflect.ValueOf(slice)
	kind := _slice.Type().Kind()

	if kind != reflect.Slice && kind != reflect.Array {
		return false
	}

	var value interface{}
	if _slice.Len() > index {
		value = _slice.Index(index).Interface()
	} else if yes {
		value = _default
	} else {
		return false
	}

	_out.Set(reflect.ValueOf(value))
	return true
}

// Same as SetValue, but if index >= len(slice), set the value of out to _default,
// and return true always.
func SetValueWithDefault(out interface{}, slice interface{}, index int, _default interface{}) bool {
	return setValue(out, slice, index, _default, true)
}

// Set the value of 'out' to 'slice[index]' and return true.
//
// Return false if the value of out can't be changed, that's, out need to be a pointer.
// Return false if slice is not a slice type or index >= len(slice).
// Panic for other cases.
func SetValue(out interface{}, slice interface{}, index int) bool {
	return setValue(out, slice, index, nil, false)
}

// Return true if value is in slice. Or false. Also reutrn false if value or
// slice is nil, or the length of slice is 0.
//
// The type of value must be consistent with the type of the element of slice.
// Or panic. If the type is the customizable struct, it MUST implement the interface
// Comparer in the package "github.com/xgfone/go-tools/compare".
func In(value interface{}, slice interface{}) bool {
	if value == nil || slice == nil {
		return false
	}

	stype := reflect.ValueOf(slice)
	if stype.Kind() == reflect.Ptr {
		stype = stype.Elem()
	}

	if stype.Kind() != reflect.Array && stype.Kind() != reflect.Slice {
		panic("The second argument is not a slice or an array")
	}

	slen := stype.Len()
	if slen == 0 {
		return false
	}

	vv := reflect.ValueOf(value)
	if stype.Index(0).Kind() != reflect.ValueOf(value).Kind() {
		panic("The type of value must be consistent with the type of the element of slice")
	}

	for i := 0; i < slen; i++ {
		v1 := vv.Interface()
		v2 := stype.Index(i).Interface()
		if compare.EQ(v1, v2) {
			return true
		}
	}

	return false
}
<commit_msg>Update the comment of the package slice.<commit_after>// Package slice gets a value from a slice and check whether a value exists in a slice.
//
// If the index is out-of-bounds, return the default value.
//
package slice

import (
	"reflect"

	"github.com/xgfone/go-tools/compare"
)

func setValue(out interface{}, slice interface{}, index int, _default interface{}, yes bool) bool {
	_out := reflect.Indirect(reflect.ValueOf(out))
	if !_out.CanSet() {
		return false
	}

	_slice := reflect.ValueOf(slice)
	kind := _slice.Type().Kind()

	if kind != reflect.Slice && kind != reflect.Array {
		return false
	}

	var value interface{}
	if _slice.Len() > index {
		value = _slice.Index(index).Interface()
	} else if yes {
		value = _default
	} else {
		return false
	}

	_out.Set(reflect.ValueOf(value))
	return true
}

// SetValueWithDefault is same as SetValue, but if index >= len(slice),
// set the value of out to _default, and return true always.
func SetValueWithDefault(out interface{}, slice interface{}, index int, _default interface{}) bool {
	return setValue(out, slice, index, _default, true)
}

// SetValue sets the value of 'out' to 'slice[index]' and return true.
//
// Return false if the value of out can't be changed, that's, out need to be a pointer.
// Return false if slice is not a slice type or index >= len(slice).
// Panic for other cases.
func SetValue(out interface{}, slice interface{}, index int) bool {
	return setValue(out, slice, index, nil, false)
}

// In returns true if value is in slice. Or false. Also reutrn false if value or
// slice is nil, or the length of slice is 0.
//
// The type of value must be consistent with the type of the element of slice.
// Or panic. If the type is the customizable struct, it MUST implement the interface
// Comparer in the package "github.com/xgfone/go-tools/compare".
func In(value interface{}, slice interface{}) bool {
	if value == nil || slice == nil {
		return false
	}

	stype := reflect.ValueOf(slice)
	if stype.Kind() == reflect.Ptr {
		stype = stype.Elem()
	}

	if stype.Kind() != reflect.Array && stype.Kind() != reflect.Slice {
		panic("The second argument is not a slice or an array")
	}

	slen := stype.Len()
	if slen == 0 {
		return false
	}

	vv := reflect.ValueOf(value)
	if stype.Index(0).Kind() != reflect.ValueOf(value).Kind() {
		panic("The type of value must be consistent with the type of the element of slice")
	}

	for i := 0; i < slen; i++ {
		v1 := vv.Interface()
		v2 := stype.Index(i).Interface()
		if compare.EQ(v1, v2) {
			return true
		}
	}

	return false
}
<|endoftext|>