<commit_before>package dns

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/golang/glog"
	"k8s.io/client-go/kubernetes"
	corev1listers "k8s.io/client-go/listers/core/v1"

	"github.com/pkg/errors"

	"github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha1"

	"github.com/jetstack/cert-manager/pkg/issuer/acme/dns/akamai"
	"github.com/jetstack/cert-manager/pkg/issuer/acme/dns/azuredns"
	"github.com/jetstack/cert-manager/pkg/issuer/acme/dns/clouddns"
	"github.com/jetstack/cert-manager/pkg/issuer/acme/dns/cloudflare"
	"github.com/jetstack/cert-manager/pkg/issuer/acme/dns/route53"
	"github.com/jetstack/cert-manager/pkg/issuer/acme/dns/util"
)

const (
	cloudDNSServiceAccountKey = "service-account.json"
)

type solver interface {
	Present(domain, token, key string) error
	CleanUp(domain, token, key string) error
	Timeout() (timeout, interval time.Duration)
}

// dnsProviderConstructors defines how each provider may be constructed.
// It is useful for mocking out a given provider since an alternate set of
// constructors may be set.
type dnsProviderConstructors struct {
	cloudDNS   func(project string, serviceAccount []byte) (*clouddns.DNSProvider, error)
	cloudFlare func(email, apikey string) (*cloudflare.DNSProvider, error)
	route53    func(accessKey, secretKey, hostedZoneID, region string, ambient bool) (*route53.DNSProvider, error)
	azureDNS   func(clientID, clientSecret, subscriptionID, tenentID, resourceGroupName, hostedZoneName string) (*azuredns.DNSProvider, error)
}

// Solver is a solver for the acme dns01 challenge.
// Given a Certificate object, it determines the correct DNS provider based on
// the certificate, and configures it based on the referenced issuer.
type Solver struct {
	issuer                  v1alpha1.GenericIssuer
	client                  kubernetes.Interface
	secretLister            corev1listers.SecretLister
	resourceNamespace       string
	dnsProviderConstructors dnsProviderConstructors
	ambientCredentials      bool
}

func (s *Solver) Present(ctx context.Context, _ *v1alpha1.Certificate, ch v1alpha1.ACMEOrderChallenge) error {
	if ch.ACMESolverConfig.DNS01 == nil {
		return fmt.Errorf("challenge dns config must be specified")
	}

	providerName := ch.ACMESolverConfig.DNS01.Provider
	if providerName == "" {
		return fmt.Errorf("dns01 challenge provider name must be set")
	}

	slv, err := s.solverForIssuerProvider(providerName)
	if err != nil {
		return err
	}

	glog.Infof("Presenting DNS01 challenge for domain %q", ch.Domain)
	return slv.Present(ch.Domain, ch.Token, ch.Key)
}

func (s *Solver) Check(ch v1alpha1.ACMEOrderChallenge) (bool, error) {
	fqdn, value, ttl := util.DNS01Record(ch.Domain, ch.Key)
	glog.Infof("Checking DNS propagation for %q using name servers: %v", ch.Domain, util.RecursiveNameservers)

	ok, err := util.PreCheckDNS(fqdn, value)
	if err != nil {
		return false, err
	}
	if !ok {
		glog.Infof("DNS record for %q not yet propagated", ch.Domain)
		return false, nil
	}

	glog.Infof("Waiting DNS record TTL (%ds) to allow propagation of DNS record for domain %q", ttl, fqdn)
	time.Sleep(time.Second * time.Duration(ttl))
	glog.Infof("ACME DNS01 validation record propagated for %q", fqdn)

	return true, nil
}

func (s *Solver) CleanUp(ctx context.Context, _ *v1alpha1.Certificate, ch v1alpha1.ACMEOrderChallenge) error {
	if ch.ACMESolverConfig.DNS01 == nil {
		return fmt.Errorf("challenge dns config must be specified")
	}

	providerName := ch.ACMESolverConfig.DNS01.Provider
	if providerName == "" {
		return fmt.Errorf("dns01 challenge provider name must be set")
	}

	slv, err := s.solverForIssuerProvider(providerName)
	if err != nil {
		return err
	}

	return slv.CleanUp(ch.Domain, ch.Token, ch.Key)
}

// returns the provider name for a given domain name by reading the acme
// configuration block on the given Certificate resource
func (s *Solver) providerForDomain(crt *v1alpha1.Certificate, domain string) (string, error) {
	var cfg *v1alpha1.ACMECertificateDNS01Config
	if cfg = crt.Spec.ACME.ConfigForDomain(domain).DNS01; cfg == nil ||
		cfg.Provider == "" ||
		s.issuer.GetSpec().ACME == nil ||
		s.issuer.GetSpec().ACME.DNS01 == nil {
		return "", fmt.Errorf("dns-01 challenge provider for domain %q is not configured. Ensure the Certificate resource configures a dns-01 provider for the domain", domain)
	}
	return cfg.Provider, nil
}

// solverForIssuerProvider returns a Solver for the given providerName.
// The providerName is the name of an ACME DNS-01 challenge provider as
// specified on the Issuer resource for the Solver.
//
// This method is exported so that only the provider name is required in order
// to obtain an instance of a Solver. This is useful when cleaning up old
// challenges after the ACME challenge configuration on the Certificate has
// been removed by the user.
func (s *Solver) solverForIssuerProvider(providerName string) (solver, error) {
	providerConfig, err := s.issuer.GetSpec().ACME.DNS01.Provider(providerName)
	if err != nil {
		return nil, err
	}

	var impl solver
	switch {
	case providerConfig.Akamai != nil:
		clientToken, err := s.loadSecretData(&providerConfig.Akamai.ClientToken)
		if err != nil {
			return nil, errors.Wrap(err, "error getting akamai client token")
		}

		clientSecret, err := s.loadSecretData(&providerConfig.Akamai.ClientSecret)
		if err != nil {
			return nil, errors.Wrap(err, "error getting akamai client secret")
		}

		accessToken, err := s.loadSecretData(&providerConfig.Akamai.AccessToken)
		if err != nil {
			return nil, errors.Wrap(err, "error getting akamai access token")
		}

		impl, err = akamai.NewDNSProvider(
			providerConfig.Akamai.ServiceConsumerDomain,
			string(clientToken),
			string(clientSecret),
			string(accessToken))
		if err != nil {
			return nil, errors.Wrap(err, "error instantiating akamai challenge solver")
		}
	case providerConfig.CloudDNS != nil:
		saSecret, err := s.secretLister.Secrets(s.resourceNamespace).Get(providerConfig.CloudDNS.ServiceAccount.Name)
		if err != nil {
			return nil, fmt.Errorf("error getting clouddns service account: %s", err.Error())
		}
		saBytes := saSecret.Data[providerConfig.CloudDNS.ServiceAccount.Key]

		impl, err = s.dnsProviderConstructors.cloudDNS(providerConfig.CloudDNS.Project, saBytes)
		if err != nil {
			return nil, fmt.Errorf("error instantiating google clouddns challenge solver: %s", err.Error())
		}
	case providerConfig.Cloudflare != nil:
		apiKeySecret, err := s.secretLister.Secrets(s.resourceNamespace).Get(providerConfig.Cloudflare.APIKey.Name)
		if err != nil {
			return nil, fmt.Errorf("error getting cloudflare service account: %s", err.Error())
		}

		email := providerConfig.Cloudflare.Email
		apiKey := string(apiKeySecret.Data[providerConfig.Cloudflare.APIKey.Key])

		impl, err = s.dnsProviderConstructors.cloudFlare(email, apiKey)
		if err != nil {
			return nil, fmt.Errorf("error instantiating cloudflare challenge solver: %s", err.Error())
		}
	case providerConfig.Route53 != nil:
		secretAccessKey := ""
		if providerConfig.Route53.SecretAccessKey.Name != "" {
			secretAccessKeySecret, err := s.secretLister.Secrets(s.resourceNamespace).Get(providerConfig.Route53.SecretAccessKey.Name)
			if err != nil {
				return nil, fmt.Errorf("error getting route53 secret access key: %s", err.Error())
			}

			secretAccessKeyBytes, ok := secretAccessKeySecret.Data[providerConfig.Route53.SecretAccessKey.Key]
			if !ok {
				return nil, fmt.Errorf("error getting route53 secret access key: key '%s' not found in secret", providerConfig.Route53.SecretAccessKey.Key)
			}
			secretAccessKey = string(secretAccessKeyBytes)
		}

		impl, err = s.dnsProviderConstructors.route53(
			strings.TrimSpace(providerConfig.Route53.AccessKeyID),
			strings.TrimSpace(secretAccessKey),
			providerConfig.Route53.HostedZoneID,
			providerConfig.Route53.Region,
			s.ambientCredentials,
		)
		if err != nil {
			return nil, fmt.Errorf("error instantiating route53 challenge solver: %s", err.Error())
		}
	case providerConfig.AzureDNS != nil:
		clientSecret, err := s.secretLister.Secrets(s.resourceNamespace).Get(providerConfig.AzureDNS.ClientSecret.Name)
		if err != nil {
			return nil, fmt.Errorf("error getting azuredns client secret: %s", err.Error())
		}

		clientSecretBytes, ok := clientSecret.Data[providerConfig.AzureDNS.ClientSecret.Key]
		if !ok {
			return nil, fmt.Errorf("error getting azure dns client secret: key '%s' not found in secret", providerConfig.AzureDNS.ClientSecret.Key)
		}

		impl, err = s.dnsProviderConstructors.azureDNS(
			providerConfig.AzureDNS.ClientID,
			string(clientSecretBytes),
			providerConfig.AzureDNS.SubscriptionID,
			providerConfig.AzureDNS.TenantID,
			providerConfig.AzureDNS.ResourceGroupName,
			providerConfig.AzureDNS.HostedZoneName,
		)
	default:
		return nil, fmt.Errorf("no dns provider config specified for provider %q", providerName)
	}

	return impl, nil
}

func NewSolver(issuer v1alpha1.GenericIssuer, client kubernetes.Interface, secretLister corev1listers.SecretLister, resourceNamespace string, ambientCredentials bool) *Solver {
	return &Solver{
		issuer,
		client,
		secretLister,
		resourceNamespace,
		dnsProviderConstructors{
			clouddns.NewDNSProviderServiceAccountBytes,
			cloudflare.NewDNSProviderCredentials,
			route53.NewDNSProvider,
			azuredns.NewDNSProviderCredentials,
		},
		ambientCredentials,
	}
}

func (s *Solver) loadSecretData(selector *v1alpha1.SecretKeySelector) ([]byte, error) {
	secret, err := s.secretLister.Secrets(s.resourceNamespace).Get(selector.Name)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to load secret with name %q", selector.Name)
	}

	if data, ok := secret.Data[selector.Key]; ok {
		return data, nil
	}

	return nil, errors.Errorf("no key %q in secret %q", selector.Key, selector.Name)
}
<commit_msg>issuer/acme/dns: remove redundant 'Error' calls<commit_after>package dns

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/golang/glog"
	"k8s.io/client-go/kubernetes"
	corev1listers "k8s.io/client-go/listers/core/v1"

	"github.com/pkg/errors"

	"github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha1"

	"github.com/jetstack/cert-manager/pkg/issuer/acme/dns/akamai"
	"github.com/jetstack/cert-manager/pkg/issuer/acme/dns/azuredns"
	"github.com/jetstack/cert-manager/pkg/issuer/acme/dns/clouddns"
	"github.com/jetstack/cert-manager/pkg/issuer/acme/dns/cloudflare"
	"github.com/jetstack/cert-manager/pkg/issuer/acme/dns/route53"
	"github.com/jetstack/cert-manager/pkg/issuer/acme/dns/util"
)

const (
	cloudDNSServiceAccountKey = "service-account.json"
)

type solver interface {
	Present(domain, token, key string) error
	CleanUp(domain, token, key string) error
	Timeout() (timeout, interval time.Duration)
}

// dnsProviderConstructors defines how each provider may be constructed.
// It is useful for mocking out a given provider since an alternate set of
// constructors may be set.
type dnsProviderConstructors struct {
	cloudDNS   func(project string, serviceAccount []byte) (*clouddns.DNSProvider, error)
	cloudFlare func(email, apikey string) (*cloudflare.DNSProvider, error)
	route53    func(accessKey, secretKey, hostedZoneID, region string, ambient bool) (*route53.DNSProvider, error)
	azureDNS   func(clientID, clientSecret, subscriptionID, tenentID, resourceGroupName, hostedZoneName string) (*azuredns.DNSProvider, error)
}

// Solver is a solver for the acme dns01 challenge.
// Given a Certificate object, it determines the correct DNS provider based on
// the certificate, and configures it based on the referenced issuer.
type Solver struct {
	issuer                  v1alpha1.GenericIssuer
	client                  kubernetes.Interface
	secretLister            corev1listers.SecretLister
	resourceNamespace       string
	dnsProviderConstructors dnsProviderConstructors
	ambientCredentials      bool
}

func (s *Solver) Present(ctx context.Context, _ *v1alpha1.Certificate, ch v1alpha1.ACMEOrderChallenge) error {
	if ch.ACMESolverConfig.DNS01 == nil {
		return fmt.Errorf("challenge dns config must be specified")
	}

	providerName := ch.ACMESolverConfig.DNS01.Provider
	if providerName == "" {
		return fmt.Errorf("dns01 challenge provider name must be set")
	}

	slv, err := s.solverForIssuerProvider(providerName)
	if err != nil {
		return err
	}

	glog.Infof("Presenting DNS01 challenge for domain %q", ch.Domain)
	return slv.Present(ch.Domain, ch.Token, ch.Key)
}

func (s *Solver) Check(ch v1alpha1.ACMEOrderChallenge) (bool, error) {
	fqdn, value, ttl := util.DNS01Record(ch.Domain, ch.Key)
	glog.Infof("Checking DNS propagation for %q using name servers: %v", ch.Domain, util.RecursiveNameservers)

	ok, err := util.PreCheckDNS(fqdn, value)
	if err != nil {
		return false, err
	}
	if !ok {
		glog.Infof("DNS record for %q not yet propagated", ch.Domain)
		return false, nil
	}

	glog.Infof("Waiting DNS record TTL (%ds) to allow propagation of DNS record for domain %q", ttl, fqdn)
	time.Sleep(time.Second * time.Duration(ttl))
	glog.Infof("ACME DNS01 validation record propagated for %q", fqdn)

	return true, nil
}

func (s *Solver) CleanUp(ctx context.Context, _ *v1alpha1.Certificate, ch v1alpha1.ACMEOrderChallenge) error {
	if ch.ACMESolverConfig.DNS01 == nil {
		return fmt.Errorf("challenge dns config must be specified")
	}

	providerName := ch.ACMESolverConfig.DNS01.Provider
	if providerName == "" {
		return fmt.Errorf("dns01 challenge provider name must be set")
	}

	slv, err := s.solverForIssuerProvider(providerName)
	if err != nil {
		return err
	}

	return slv.CleanUp(ch.Domain, ch.Token, ch.Key)
}

// returns the provider name for a given domain name by reading the acme
// configuration block on the given Certificate resource
func (s *Solver) providerForDomain(crt *v1alpha1.Certificate, domain string) (string, error) {
	var cfg *v1alpha1.ACMECertificateDNS01Config
	if cfg = crt.Spec.ACME.ConfigForDomain(domain).DNS01; cfg == nil ||
		cfg.Provider == "" ||
		s.issuer.GetSpec().ACME == nil ||
		s.issuer.GetSpec().ACME.DNS01 == nil {
		return "", fmt.Errorf("dns-01 challenge provider for domain %q is not configured. Ensure the Certificate resource configures a dns-01 provider for the domain", domain)
	}
	return cfg.Provider, nil
}

// solverForIssuerProvider returns a Solver for the given providerName.
// The providerName is the name of an ACME DNS-01 challenge provider as
// specified on the Issuer resource for the Solver.
//
// This method is exported so that only the provider name is required in order
// to obtain an instance of a Solver. This is useful when cleaning up old
// challenges after the ACME challenge configuration on the Certificate has
// been removed by the user.
func (s *Solver) solverForIssuerProvider(providerName string) (solver, error) {
	providerConfig, err := s.issuer.GetSpec().ACME.DNS01.Provider(providerName)
	if err != nil {
		return nil, err
	}

	var impl solver
	switch {
	case providerConfig.Akamai != nil:
		clientToken, err := s.loadSecretData(&providerConfig.Akamai.ClientToken)
		if err != nil {
			return nil, errors.Wrap(err, "error getting akamai client token")
		}

		clientSecret, err := s.loadSecretData(&providerConfig.Akamai.ClientSecret)
		if err != nil {
			return nil, errors.Wrap(err, "error getting akamai client secret")
		}

		accessToken, err := s.loadSecretData(&providerConfig.Akamai.AccessToken)
		if err != nil {
			return nil, errors.Wrap(err, "error getting akamai access token")
		}

		impl, err = akamai.NewDNSProvider(
			providerConfig.Akamai.ServiceConsumerDomain,
			string(clientToken),
			string(clientSecret),
			string(accessToken))
		if err != nil {
			return nil, errors.Wrap(err, "error instantiating akamai challenge solver")
		}
	case providerConfig.CloudDNS != nil:
		saSecret, err := s.secretLister.Secrets(s.resourceNamespace).Get(providerConfig.CloudDNS.ServiceAccount.Name)
		if err != nil {
			return nil, fmt.Errorf("error getting clouddns service account: %s", err)
		}
		saBytes := saSecret.Data[providerConfig.CloudDNS.ServiceAccount.Key]

		impl, err = s.dnsProviderConstructors.cloudDNS(providerConfig.CloudDNS.Project, saBytes)
		if err != nil {
			return nil, fmt.Errorf("error instantiating google clouddns challenge solver: %s", err)
		}
	case providerConfig.Cloudflare != nil:
		apiKeySecret, err := s.secretLister.Secrets(s.resourceNamespace).Get(providerConfig.Cloudflare.APIKey.Name)
		if err != nil {
			return nil, fmt.Errorf("error getting cloudflare service account: %s", err)
		}

		email := providerConfig.Cloudflare.Email
		apiKey := string(apiKeySecret.Data[providerConfig.Cloudflare.APIKey.Key])

		impl, err = s.dnsProviderConstructors.cloudFlare(email, apiKey)
		if err != nil {
			return nil, fmt.Errorf("error instantiating cloudflare challenge solver: %s", err)
		}
	case providerConfig.Route53 != nil:
		secretAccessKey := ""
		if providerConfig.Route53.SecretAccessKey.Name != "" {
			secretAccessKeySecret, err := s.secretLister.Secrets(s.resourceNamespace).Get(providerConfig.Route53.SecretAccessKey.Name)
			if err != nil {
				return nil, fmt.Errorf("error getting route53 secret access key: %s", err)
			}

			secretAccessKeyBytes, ok := secretAccessKeySecret.Data[providerConfig.Route53.SecretAccessKey.Key]
			if !ok {
				return nil, fmt.Errorf("error getting route53 secret access key: key '%s' not found in secret", providerConfig.Route53.SecretAccessKey.Key)
			}
			secretAccessKey = string(secretAccessKeyBytes)
		}

		impl, err = s.dnsProviderConstructors.route53(
			strings.TrimSpace(providerConfig.Route53.AccessKeyID),
			strings.TrimSpace(secretAccessKey),
			providerConfig.Route53.HostedZoneID,
			providerConfig.Route53.Region,
			s.ambientCredentials,
		)
		if err != nil {
			return nil, fmt.Errorf("error instantiating route53 challenge solver: %s", err)
		}
	case providerConfig.AzureDNS != nil:
		clientSecret, err := s.secretLister.Secrets(s.resourceNamespace).Get(providerConfig.AzureDNS.ClientSecret.Name)
		if err != nil {
			return nil, fmt.Errorf("error getting azuredns client secret: %s", err)
		}

		clientSecretBytes, ok := clientSecret.Data[providerConfig.AzureDNS.ClientSecret.Key]
		if !ok {
			return nil, fmt.Errorf("error getting azure dns client secret: key '%s' not found in secret", providerConfig.AzureDNS.ClientSecret.Key)
		}

		impl, err = s.dnsProviderConstructors.azureDNS(
			providerConfig.AzureDNS.ClientID,
			string(clientSecretBytes),
			providerConfig.AzureDNS.SubscriptionID,
			providerConfig.AzureDNS.TenantID,
			providerConfig.AzureDNS.ResourceGroupName,
			providerConfig.AzureDNS.HostedZoneName,
		)
	default:
		return nil, fmt.Errorf("no dns provider config specified for provider %q", providerName)
	}

	return impl, nil
}

func NewSolver(issuer v1alpha1.GenericIssuer, client kubernetes.Interface, secretLister corev1listers.SecretLister, resourceNamespace string, ambientCredentials bool) *Solver {
	return &Solver{
		issuer,
		client,
		secretLister,
		resourceNamespace,
		dnsProviderConstructors{
			clouddns.NewDNSProviderServiceAccountBytes,
			cloudflare.NewDNSProviderCredentials,
			route53.NewDNSProvider,
			azuredns.NewDNSProviderCredentials,
		},
		ambientCredentials,
	}
}

func (s *Solver) loadSecretData(selector *v1alpha1.SecretKeySelector) ([]byte, error) {
	secret, err := s.secretLister.Secrets(s.resourceNamespace).Get(selector.Name)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to load secret with name %q", selector.Name)
	}

	if data, ok := secret.Data[selector.Key]; ok {
		return data, nil
	}

	return nil, errors.Errorf("no key %q in secret %q", selector.Key, selector.Name)
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tunneler

import (
	"context"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"sync/atomic"
	"time"

	"k8s.io/apimachinery/pkg/util/clock"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/kubernetes/pkg/ssh"
	utilfile "k8s.io/kubernetes/pkg/util/file"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus"
)

type InstallSSHKey func(ctx context.Context, user string, data []byte) error

type AddressFunc func() (addresses []string, err error)

type Tunneler interface {
	Run(AddressFunc)
	Stop()
	Dial(net, addr string) (net.Conn, error)
	SecondsSinceSync() int64
	SecondsSinceSSHKeySync() int64
}

// TunnelSyncHealthChecker returns a health func that indicates if a tunneler is healthy.
// It's compatible with healthz.NamedCheck
func TunnelSyncHealthChecker(tunneler Tunneler) func(req *http.Request) error {
	return func(req *http.Request) error {
		if tunneler == nil {
			return nil
		}
		lag := tunneler.SecondsSinceSync()
		if lag > 600 {
			return fmt.Errorf("Tunnel sync is taking too long: %d", lag)
		}
		sshKeyLag := tunneler.SecondsSinceSSHKeySync()
		if sshKeyLag > 600 {
			return fmt.Errorf("SSHKey sync is taking too long: %d", sshKeyLag)
		}
		return nil
	}
}

type SSHTunneler struct {
	// Important: Since these two int64 fields are using sync/atomic, they have to be at the top of the struct due to a bug on 32-bit platforms
	// See: https://golang.org/pkg/sync/atomic/ for more information
	lastSync       int64 // Seconds since Epoch
	lastSSHKeySync int64 // Seconds since Epoch

	SSHUser        string
	SSHKeyfile     string
	InstallSSHKey  InstallSSHKey
	HealthCheckURL *url.URL

	tunnels        *ssh.SSHTunnelList
	lastSyncMetric prometheus.GaugeFunc
	clock          clock.Clock

	getAddresses AddressFunc
	stopChan     chan struct{}
}

func New(sshUser, sshKeyfile string, healthCheckURL *url.URL, installSSHKey InstallSSHKey) Tunneler {
	return &SSHTunneler{
		SSHUser:        sshUser,
		SSHKeyfile:     sshKeyfile,
		InstallSSHKey:  installSSHKey,
		HealthCheckURL: healthCheckURL,
		clock:          clock.RealClock{},
	}
}

// Run establishes tunnel loops and returns
func (c *SSHTunneler) Run(getAddresses AddressFunc) {
	if c.stopChan != nil {
		return
	}
	c.stopChan = make(chan struct{})

	// Save the address getter
	if getAddresses != nil {
		c.getAddresses = getAddresses
	}

	// Usernames are capped @ 32
	if len(c.SSHUser) > 32 {
		glog.Warning("SSH User is too long, truncating to 32 chars")
		c.SSHUser = c.SSHUser[0:32]
	}
	glog.Infof("Setting up proxy: %s %s", c.SSHUser, c.SSHKeyfile)

	// public keyfile is written last, so check for that.
	publicKeyFile := c.SSHKeyfile + ".pub"
	exists, err := utilfile.FileExists(publicKeyFile)
	if err != nil {
		glog.Errorf("Error detecting if key exists: %v", err)
	} else if !exists {
		glog.Infof("Key doesn't exist, attempting to create")
		if err := generateSSHKey(c.SSHKeyfile, publicKeyFile); err != nil {
			glog.Errorf("Failed to create key pair: %v", err)
		}
	}

	c.tunnels = ssh.NewSSHTunnelList(c.SSHUser, c.SSHKeyfile, c.HealthCheckURL, c.stopChan)
	// Sync loop to ensure that the SSH key has been installed.
	c.lastSSHKeySync = c.clock.Now().Unix()
	c.installSSHKeySyncLoop(c.SSHUser, publicKeyFile)
	// Sync tunnelList w/ nodes.
	c.lastSync = c.clock.Now().Unix()
	c.nodesSyncLoop()
}

// Stop gracefully shuts down the tunneler
func (c *SSHTunneler) Stop() {
	if c.stopChan != nil {
		close(c.stopChan)
		c.stopChan = nil
	}
}

func (c *SSHTunneler) Dial(net, addr string) (net.Conn, error) {
	return c.tunnels.Dial(net, addr)
}

func (c *SSHTunneler) SecondsSinceSync() int64 {
	now := c.clock.Now().Unix()
	then := atomic.LoadInt64(&c.lastSync)
	return now - then
}

func (c *SSHTunneler) SecondsSinceSSHKeySync() int64 {
	now := c.clock.Now().Unix()
	then := atomic.LoadInt64(&c.lastSSHKeySync)
	return now - then
}

func (c *SSHTunneler) installSSHKeySyncLoop(user, publicKeyfile string) {
	go wait.Until(func() {
		if c.InstallSSHKey == nil {
			glog.Error("Won't attempt to install ssh key: InstallSSHKey function is nil")
			return
		}
		key, err := ssh.ParsePublicKeyFromFile(publicKeyfile)
		if err != nil {
			glog.Errorf("Failed to load public key: %v", err)
			return
		}
		keyData, err := ssh.EncodeSSHKey(key)
		if err != nil {
			glog.Errorf("Failed to encode public key: %v", err)
			return
		}
		if err := c.InstallSSHKey(context.TODO(), user, keyData); err != nil {
			glog.Errorf("Failed to install ssh key: %v", err)
			return
		}
		atomic.StoreInt64(&c.lastSSHKeySync, c.clock.Now().Unix())
	}, 5*time.Minute, c.stopChan)
}

// nodesSyncLoop lists nodes every 15 seconds, calling Update() on the TunnelList
// each time (Update() is a noop if no changes are necessary).
func (c *SSHTunneler) nodesSyncLoop() {
	// TODO (cjcullen) make this watch.
	go wait.Until(func() {
		addrs, err := c.getAddresses()
		glog.V(4).Infof("Calling update w/ addrs: %v", addrs)
		if err != nil {
			glog.Errorf("Failed to getAddresses: %v", err)
		}
		c.tunnels.Update(addrs)
		atomic.StoreInt64(&c.lastSync, c.clock.Now().Unix())
	}, 15*time.Second, c.stopChan)
}

func generateSSHKey(privateKeyfile, publicKeyfile string) error {
	private, public, err := ssh.GenerateKey(2048)
	if err != nil {
		return err
	}
	// If private keyfile already exists, we must have only made it halfway
	// through last time, so delete it.
	exists, err := utilfile.FileExists(privateKeyfile)
	if err != nil {
		glog.Errorf("Error detecting if private key exists: %v", err)
	} else if exists {
		glog.Infof("Private key exists, but public key does not")
		if err := os.Remove(privateKeyfile); err != nil {
			glog.Errorf("Failed to remove stale private key: %v", err)
		}
	}
	if err := ioutil.WriteFile(privateKeyfile, ssh.EncodePrivateKey(private), 0600); err != nil {
		return err
	}
	publicKeyBytes, err := ssh.EncodePublicKey(public)
	if err != nil {
		return err
	}
	if err := ioutil.WriteFile(publicKeyfile+".tmp", publicKeyBytes, 0600); err != nil {
		return err
	}
	return os.Rename(publicKeyfile+".tmp", publicKeyfile)
}
<commit_msg>Increase allowed lag for ssh key sync loop for tunneler<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tunneler

import (
	"context"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"sync/atomic"
	"time"

	"k8s.io/apimachinery/pkg/util/clock"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/kubernetes/pkg/ssh"
	utilfile "k8s.io/kubernetes/pkg/util/file"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus"
)

type InstallSSHKey func(ctx context.Context, user string, data []byte) error

type AddressFunc func() (addresses []string, err error)

type Tunneler interface {
	Run(AddressFunc)
	Stop()
	Dial(net, addr string) (net.Conn, error)
	SecondsSinceSync() int64
	SecondsSinceSSHKeySync() int64
}

// TunnelSyncHealthChecker returns a health func that indicates if a tunneler is healthy.
// It's compatible with healthz.NamedCheck
func TunnelSyncHealthChecker(tunneler Tunneler) func(req *http.Request) error {
	return func(req *http.Request) error {
		if tunneler == nil {
			return nil
		}
		lag := tunneler.SecondsSinceSync()
		if lag > 600 {
			return fmt.Errorf("Tunnel sync is taking too long: %d", lag)
		}
		sshKeyLag := tunneler.SecondsSinceSSHKeySync()
		// Since we are syncing ssh-keys every 5 minutes, the allowed
		// lag since last sync should be more than 2x higher than that
		// to allow for single failure, which can always happen.
		// For now set it to 3x, which is 15 minutes.
		// For more details see: http://pr.k8s.io/59347
		if sshKeyLag > 900 {
			return fmt.Errorf("SSHKey sync is taking too long: %d", sshKeyLag)
		}
		return nil
	}
}

type SSHTunneler struct {
	// Important: Since these two int64 fields are using sync/atomic, they have to be at the top of the struct due to a bug on 32-bit platforms
	// See: https://golang.org/pkg/sync/atomic/ for more information
	lastSync       int64 // Seconds since Epoch
	lastSSHKeySync int64 // Seconds since Epoch

	SSHUser        string
	SSHKeyfile     string
	InstallSSHKey  InstallSSHKey
	HealthCheckURL *url.URL

	tunnels        *ssh.SSHTunnelList
	lastSyncMetric prometheus.GaugeFunc
	clock          clock.Clock

	getAddresses AddressFunc
	stopChan     chan struct{}
}

func New(sshUser, sshKeyfile string, healthCheckURL *url.URL, installSSHKey InstallSSHKey) Tunneler {
	return &SSHTunneler{
		SSHUser:        sshUser,
		SSHKeyfile:     sshKeyfile,
		InstallSSHKey:  installSSHKey,
		HealthCheckURL: healthCheckURL,
		clock:          clock.RealClock{},
	}
}

// Run establishes tunnel loops and returns
func (c *SSHTunneler) Run(getAddresses AddressFunc) {
	if c.stopChan != nil {
		return
	}
	c.stopChan = make(chan struct{})

	// Save the address getter
	if getAddresses != nil {
		c.getAddresses = getAddresses
	}

	// Usernames are capped @ 32
	if len(c.SSHUser) > 32 {
		glog.Warning("SSH User is too long, truncating to 32 chars")
		c.SSHUser = c.SSHUser[0:32]
	}
	glog.Infof("Setting up proxy: %s %s", c.SSHUser, c.SSHKeyfile)

	// public keyfile is written last, so check for that.
	publicKeyFile := c.SSHKeyfile + ".pub"
	exists, err := utilfile.FileExists(publicKeyFile)
	if err != nil {
		glog.Errorf("Error detecting if key exists: %v", err)
	} else if !exists {
		glog.Infof("Key doesn't exist, attempting to create")
		if err := generateSSHKey(c.SSHKeyfile, publicKeyFile); err != nil {
			glog.Errorf("Failed to create key pair: %v", err)
		}
	}

	c.tunnels = ssh.NewSSHTunnelList(c.SSHUser, c.SSHKeyfile, c.HealthCheckURL, c.stopChan)
	// Sync loop to ensure that the SSH key has been installed.
	c.lastSSHKeySync = c.clock.Now().Unix()
	c.installSSHKeySyncLoop(c.SSHUser, publicKeyFile)
	// Sync tunnelList w/ nodes.
	c.lastSync = c.clock.Now().Unix()
	c.nodesSyncLoop()
}

// Stop gracefully shuts down the tunneler
func (c *SSHTunneler) Stop() {
	if c.stopChan != nil {
		close(c.stopChan)
		c.stopChan = nil
	}
}

func (c *SSHTunneler) Dial(net, addr string) (net.Conn, error) {
	return c.tunnels.Dial(net, addr)
}

func (c *SSHTunneler) SecondsSinceSync() int64 {
	now := c.clock.Now().Unix()
	then := atomic.LoadInt64(&c.lastSync)
	return now - then
}

func (c *SSHTunneler) SecondsSinceSSHKeySync() int64 {
	now := c.clock.Now().Unix()
	then := atomic.LoadInt64(&c.lastSSHKeySync)
	return now - then
}

func (c *SSHTunneler) installSSHKeySyncLoop(user, publicKeyfile string) {
	go wait.Until(func() {
		if c.InstallSSHKey == nil {
			glog.Error("Won't attempt to install ssh key: InstallSSHKey function is nil")
			return
		}
		key, err := ssh.ParsePublicKeyFromFile(publicKeyfile)
		if err != nil {
			glog.Errorf("Failed to load public key: %v", err)
			return
		}
		keyData, err := ssh.EncodeSSHKey(key)
		if err != nil {
			glog.Errorf("Failed to encode public key: %v", err)
			return
		}
		if err := c.InstallSSHKey(context.TODO(), user, keyData); err != nil {
			glog.Errorf("Failed to install ssh key: %v", err)
			return
		}
		atomic.StoreInt64(&c.lastSSHKeySync, c.clock.Now().Unix())
	}, 5*time.Minute, c.stopChan)
}

// nodesSyncLoop lists nodes every 15 seconds, calling Update() on the TunnelList
// each time (Update() is a noop if no changes are necessary).
func (c *SSHTunneler) nodesSyncLoop() {
	// TODO (cjcullen) make this watch.
	go wait.Until(func() {
		addrs, err := c.getAddresses()
		glog.V(4).Infof("Calling update w/ addrs: %v", addrs)
		if err != nil {
			glog.Errorf("Failed to getAddresses: %v", err)
		}
		c.tunnels.Update(addrs)
		atomic.StoreInt64(&c.lastSync, c.clock.Now().Unix())
	}, 15*time.Second, c.stopChan)
}

func generateSSHKey(privateKeyfile, publicKeyfile string) error {
	private, public, err := ssh.GenerateKey(2048)
	if err != nil {
		return err
	}
	// If private keyfile already exists, we must have only made it halfway
	// through last time, so delete it.
	exists, err := utilfile.FileExists(privateKeyfile)
	if err != nil {
		glog.Errorf("Error detecting if private key exists: %v", err)
	} else if exists {
		glog.Infof("Private key exists, but public key does not")
		if err := os.Remove(privateKeyfile); err != nil {
			glog.Errorf("Failed to remove stale private key: %v", err)
		}
	}
	if err := ioutil.WriteFile(privateKeyfile, ssh.EncodePrivateKey(private), 0600); err != nil {
		return err
	}
	publicKeyBytes, err := ssh.EncodePublicKey(public)
	if err != nil {
		return err
	}
	if err := ioutil.WriteFile(publicKeyfile+".tmp", publicKeyBytes, 0600); err != nil {
		return err
	}
	return os.Rename(publicKeyfile+".tmp", publicKeyfile)
}
<|endoftext|>


<commit_before>// Copyright ©2016 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package native

import (
	"math"

	"github.com/gonum/blas"
	"github.com/gonum/blas/blas64"
	"github.com/gonum/lapack"
)

// Dsteqr computes the eigenvalues and optionally the eigenvectors of a symmetric
// tridiagonal matrix using the implicit QL or QR method. The eigenvectors of a
// full or band symmetric matrix can also be found if Dsytrd, Dsptrd, or Dsbtrd
// have been used to reduce this matrix to tridiagonal form.
//
// d, on entry, contains the diagonal elements of the tridiagonal matrix. On exit,
// d contains the eigenvalues in ascending order. d must have length n and
// Dsteqr will panic otherwise.
//
// e, on entry, contains the off-diagonal elements of the tridiagonal matrix on
// entry, and is overwritten during the call to Dsteqr. e must have length n-1 and
// Dsteqr will panic otherwise.
//
// z, on entry, contains the n×n orthogonal matrix used in the reduction to
// tridiagonal form if compz == lapack.EigDecomp. On exit, if
// compz == lapack.EigBoth, z contains the orthonormal eigenvectors of the
// original symmetric matrix, and if compz == lapack.EigDecomp, z contains the
// orthonormal eigenvectors of the symmetric tridiagonal matrix. z is not used
// if compz == lapack.EigValueOnly.
//
// work must have length at least max(1, 2*n-2) if the eigenvectors are computed,
// and Dsteqr will panic otherwise.
func (impl Implementation) Dsteqr(compz lapack.EigComp, n int, d, e, z []float64, ldz int, work []float64) (ok bool) {
	if len(d) < n {
		panic(badD)
	}
	if len(e) < n-1 {
		panic(badE)
	}
	if compz != lapack.EigValueOnly && compz != lapack.EigBoth && compz != lapack.EigDecomp {
		panic(badEigComp)
	}
	if compz != lapack.EigValueOnly {
		if len(work) < max(1, 2*n-2) {
			panic(badWork)
		}
		checkMatrix(n, n, z, ldz)
	}

	var icompz int
	if compz == lapack.EigDecomp {
		icompz = 1
	} else if compz == lapack.EigBoth {
		icompz = 2
	}

	if n == 0 {
		return true
	}
	if n == 1 {
		if icompz == 2 {
			z[0] = 1
		}
		return true
	}

	bi := blas64.Implementation()

	eps := dlamchE
	eps2 := eps * eps
	safmin := dlamchS
	safmax := 1 / safmin
	ssfmax := math.Sqrt(safmax) / 3
	ssfmin := math.Sqrt(safmin) / eps2

	// Compute the eigenvalues and eigenvectors of the tridiagonal matrix.
	if icompz == 2 {
		impl.Dlaset(blas.All, n, n, 0, 1, z, ldz)
	}
	const maxit = 30
	nmaxit := n * maxit

	jtot := 0

	// Determine where the matrix splits and choose QL or QR iteration for each
	// block, according to whether top or bottom diagonal element is smaller.
	l1 := 0
	nm1 := n - 1

	type scaletype int
	const (
		none scaletype = iota
		down
		up
	)
	var iscale scaletype

	for {
		if l1 > n-1 {
			// Order eigenvalues and eigenvectors.
			if icompz == 0 {
				impl.Dlasrt(lapack.SortIncreasing, n, d)
			} else {
				// TODO(btracey): Consider replacing this sort with a call to sort.Sort.
				for ii := 1; ii < n; ii++ {
					i := ii - 1
					k := i
					p := d[i]
					for j := ii; j < n; j++ {
						if d[j] < p {
							k = j
							p = d[j]
						}
					}
					if k != i {
						d[k] = d[i]
						d[i] = p
						bi.Dswap(n, z[i:], ldz, z[k:], ldz)
					}
				}
			}
			return true
		}
		if l1 > 0 {
			e[l1-1] = 0
		}
		var m int
		if l1 <= nm1 {
			for m = l1; m < nm1; m++ {
				test := math.Abs(e[m])
				if test == 0 {
					break
				}
				if test <= (math.Sqrt(math.Abs(d[m]))*math.Sqrt(math.Abs(d[m+1])))*eps {
					e[m] = 0
					break
				}
			}
		}
		l := l1
		lsv := l
		lend := m
		lendsv := lend
		l1 = m + 1
		if lend == l {
			continue
		}

		// Scale submatrix in rows and columns L to Lend
		anorm := impl.Dlanst(lapack.MaxAbs, lend-l+1, d[l:], e[l:])
		switch {
		case anorm == 0:
			continue
		case anorm > ssfmax:
			iscale = down
			// TODO(btracey): Why is lda n?
			impl.Dlascl(lapack.General, 0, 0, anorm, ssfmax, lend-l+1, 1, d[l:], n)
			impl.Dlascl(lapack.General, 0, 0, anorm, ssfmax, lend-l, 1, e[l:], n)
		case anorm < ssfmin:
			iscale = up
			// TODO(btracey): Why is lda n?
			impl.Dlascl(lapack.General, 0, 0, anorm, ssfmin, lend-l+1, 1, d[l:], n)
			impl.Dlascl(lapack.General, 0, 0, anorm, ssfmin, lend-l, 1, e[l:], n)
		}

		// Choose between QL and QR.
		if math.Abs(d[lend]) < math.Abs(d[l]) {
			lend = lsv
			l = lendsv
		}
		if lend > l {
			// QL Iteration. Look for small subdiagonal element.
			for {
				if l != lend {
					for m = l; m < lend; m++ {
						v := math.Abs(e[m])
						if v*v <= (eps2*math.Abs(d[m]))*math.Abs(d[m+1])+safmin {
							break
						}
					}
				} else {
					m = lend
				}
				if m < lend {
					e[m] = 0
				}
				p := d[l]
				if m == l {
					// Eigenvalue found.
					l++
					if l > lend {
						break
					}
					continue
				}

				// If remaining matrix is 2×2, use Dlae2 to compute its eigensystem.
				if m == l+1 {
					if icompz > 0 {
						d[l], d[l+1], work[l], work[n-1+l] = impl.Dlaev2(d[l], e[l], d[l+1])
						impl.Dlasr(blas.Right, lapack.Variable, lapack.Backward,
							n, 2, work[l:], work[n-1+l:], z[l:], ldz)
					} else {
						d[l], d[l+1] = impl.Dlae2(d[l], e[l], d[l+1])
					}
					e[l] = 0
					l += 2
					if l > lend {
						break
					}
					continue
				}

				if jtot == nmaxit {
					break
				}
				jtot++

				// Form shift
				g := (d[l+1] - p) / (2 * e[l])
				r := impl.Dlapy2(g, 1)
				g = d[m] - p + e[l]/(g+math.Copysign(r, g))
				s := 1.0
				c := 1.0
				p = 0.0

				// Inner loop
				for i := m - 1; i >= l; i-- {
					f := s * e[i]
					b := c * e[i]
					c, s, r = impl.Dlartg(g, f)
					if i != m-1 {
						e[i+1] = r
					}
					g = d[i+1] - p
					r = (d[i]-g)*s + 2*c*b
					p = s * r
					d[i+1] = g + p
					g = c*r - b

					// If eigenvectors are desired, then save rotations.
					if icompz > 0 {
						work[i] = c
						work[n-1+i] = -s
					}
				}
				// If eigenvectors are desired, then apply saved rotations.
				if icompz > 0 {
					mm := m - l + 1
					impl.Dlasr(blas.Right, lapack.Variable, lapack.Backward,
						n, mm, work[l:], work[n-1+l:], z[l:], ldz)
				}
				d[l] -= p
				e[l] = g
			}
		} else {
			// QR Iteration.
			// Look for small superdiagonal element.
			for {
				if l != lend {
					for m = l; m > lend; m-- {
						v := math.Abs(e[m-1])
						if v*v <= (eps2*math.Abs(d[m])*math.Abs(d[m-1]) + safmin) {
							break
						}
					}
				} else {
					m = lend
				}
				if m > lend {
					e[m-1] = 0
				}
				p := d[l]
				if m == l {
					// Eigenvalue found
					l--
					if l < lend {
						break
					}
					continue
				}

				// If remaining matrix is 2×2, use Dlae2 to compute its eigenvalues.
				if m == l-1 {
					if icompz > 0 {
						d[l-1], d[l], work[m], work[n-1+m] = impl.Dlaev2(d[l-1], e[l-1], d[l])
						impl.Dlasr(blas.Right, lapack.Variable, lapack.Forward,
							n, 2, work[m:], work[n-1+m:], z[l-1:], ldz)
					} else {
						d[l-1], d[l] = impl.Dlae2(d[l-1], e[l-1], d[l])
					}
					e[l-1] = 0
					l -= 2
					if l < lend {
						break
					}
					continue
				}
				if jtot == nmaxit {
					break
				}
				jtot++

				// Form shift.
				g := (d[l-1] - p) / (2 * e[l-1])
				r := impl.Dlapy2(g, 1)
				g = d[m] - p + (e[l-1])/(g+math.Copysign(r, g))
				s := 1.0
				c := 1.0
				p = 0.0

				// Inner loop.
				for i := m; i < l; i++ {
					f := s * e[i]
					b := c * e[i]
					c, s, r = impl.Dlartg(g, f)
					if i != m {
						e[i-1] = r
					}
					g = d[i] - p
					r = (d[i+1]-g)*s + 2*c*b
					p = s * r
					d[i] = g + p
					g = c*r - b

					// If eigenvectors are desired, then save rotations.
					if icompz > 0 {
						work[i] = c
						work[n-1+i] = s
					}
				}

				// If eigenvectors are desired, then apply saved rotations.
				if icompz > 0 {
					mm := l - m + 1
					impl.Dlasr(blas.Right, lapack.Variable, lapack.Forward,
						n, mm, work[m:], work[n-1+m:], z[m:], ldz)
				}
				d[l] -= p
				e[l-1] = g
			}
		}

		// Undo scaling if necessary.
		switch iscale {
		case down:
			impl.Dlascl(lapack.General, 0, 0, ssfmax, anorm, lendsv-lsv+1, 1, d[lsv:], n)
			impl.Dlascl(lapack.General, 0, 0, ssfmax, anorm, lendsv-lsv, 1, e[lsv:], n)
		case up:
			impl.Dlascl(lapack.General, 0, 0, ssfmin, anorm, lendsv-lsv+1, 1, e[lsv:], n)
			impl.Dlascl(lapack.General, 0, 0, ssfmin, anorm, lendsv-lsv, 1, e[lsv:], n)
		}

		// Check for no convergence to an eigenvalue after a total of n*maxit iterations.
		if jtot >= nmaxit {
			break
		}
	}
	for i := 0; i < n-1; i++ {
		if e[i] != 0 {
			return false
		}
	}
	return true
}
<commit_msg>native: fix scale call<commit_after>// Copyright ©2016 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package native

import (
	"math"

	"github.com/gonum/blas"
	"github.com/gonum/blas/blas64"
	"github.com/gonum/lapack"
)

// Dsteqr computes the eigenvalues and optionally the eigenvectors of a symmetric
// tridiagonal matrix using the implicit QL or QR method. The eigenvectors of a
// full or band symmetric matrix can also be found if Dsytrd, Dsptrd, or Dsbtrd
// have been used to reduce this matrix to tridiagonal form.
//
// d, on entry, contains the diagonal elements of the tridiagonal matrix. On exit,
// d contains the eigenvalues in ascending order. d must have length n and
// Dsteqr will panic otherwise.
//
// e, on entry, contains the off-diagonal elements of the tridiagonal matrix on
// entry, and is overwritten during the call to Dsteqr. e must have length n-1 and
// Dsteqr will panic otherwise.
//
// z, on entry, contains the n×n orthogonal matrix used in the reduction to
// tridiagonal form if compz == lapack.EigDecomp. On exit, if
// compz == lapack.EigBoth, z contains the orthonormal eigenvectors of the
// original symmetric matrix, and if compz == lapack.EigDecomp, z contains the
// orthonormal eigenvectors of the symmetric tridiagonal matrix. z is not used
// if compz == lapack.EigValueOnly.
//
// work must have length at least max(1, 2*n-2) if the eigenvectors are computed,
// and Dsteqr will panic otherwise.
func (impl Implementation) Dsteqr(compz lapack.EigComp, n int, d, e, z []float64, ldz int, work []float64) (ok bool) {
	if len(d) < n {
		panic(badD)
	}
	if len(e) < n-1 {
		panic(badE)
	}
	if compz != lapack.EigValueOnly && compz != lapack.EigBoth && compz != lapack.EigDecomp {
		panic(badEigComp)
	}
	if compz != lapack.EigValueOnly {
		if len(work) < max(1, 2*n-2) {
			panic(badWork)
		}
		checkMatrix(n, n, z, ldz)
	}

	var icompz int
	if compz == lapack.EigDecomp {
		icompz = 1
	} else if compz == lapack.EigBoth {
		icompz = 2
	}

	if n == 0 {
		return true
	}
	if n == 1 {
		if icompz == 2 {
			z[0] = 1
		}
		return true
	}

	bi := blas64.Implementation()

	eps := dlamchE
	eps2 := eps * eps
	safmin := dlamchS
	safmax := 1 / safmin
	ssfmax := math.Sqrt(safmax) / 3
	ssfmin := math.Sqrt(safmin) / eps2

	// Compute the eigenvalues and eigenvectors of the tridiagonal matrix.
	if icompz == 2 {
		impl.Dlaset(blas.All, n, n, 0, 1, z, ldz)
	}
	const maxit = 30
	nmaxit := n * maxit

	jtot := 0

	// Determine where the matrix splits and choose QL or QR iteration for each
	// block, according to whether top or bottom diagonal element is smaller.
	l1 := 0
	nm1 := n - 1

	type scaletype int
	const (
		none scaletype = iota
		down
		up
	)
	var iscale scaletype

	for {
		if l1 > n-1 {
			// Order eigenvalues and eigenvectors.
			if icompz == 0 {
				impl.Dlasrt(lapack.SortIncreasing, n, d)
			} else {
				// TODO(btracey): Consider replacing this sort with a call to sort.Sort.
				for ii := 1; ii < n; ii++ {
					i := ii - 1
					k := i
					p := d[i]
					for j := ii; j < n; j++ {
						if d[j] < p {
							k = j
							p = d[j]
						}
					}
					if k != i {
						d[k] = d[i]
						d[i] = p
						bi.Dswap(n, z[i:], ldz, z[k:], ldz)
					}
				}
			}
			return true
		}
		if l1 > 0 {
			e[l1-1] = 0
		}
		var m int
		if l1 <= nm1 {
			for m = l1; m < nm1; m++ {
				test := math.Abs(e[m])
				if test == 0 {
					break
				}
				if test <= (math.Sqrt(math.Abs(d[m]))*math.Sqrt(math.Abs(d[m+1])))*eps {
					e[m] = 0
					break
				}
			}
		}
		l := l1
		lsv := l
		lend := m
		lendsv := lend
		l1 = m + 1
		if lend == l {
			continue
		}

		// Scale submatrix in rows and columns L to Lend
		anorm := impl.Dlanst(lapack.MaxAbs, lend-l+1, d[l:], e[l:])
		switch {
		case anorm == 0:
			continue
		case anorm > ssfmax:
			iscale = down
			// TODO(btracey): Why is lda n?
			impl.Dlascl(lapack.General, 0, 0, anorm, ssfmax, lend-l+1, 1, d[l:], n)
			impl.Dlascl(lapack.General, 0, 0, anorm, ssfmax, lend-l, 1, e[l:], n)
		case anorm < ssfmin:
			iscale = up
			// TODO(btracey): Why is lda n?
			impl.Dlascl(lapack.General, 0, 0, anorm, ssfmin, lend-l+1, 1, d[l:], n)
			impl.Dlascl(lapack.General, 0, 0, anorm, ssfmin, lend-l, 1, e[l:], n)
		}

		// Choose between QL and QR.
		if math.Abs(d[lend]) < math.Abs(d[l]) {
			lend = lsv
			l = lendsv
		}
		if lend > l {
			// QL Iteration. Look for small subdiagonal element.
			for {
				if l != lend {
					for m = l; m < lend; m++ {
						v := math.Abs(e[m])
						if v*v <= (eps2*math.Abs(d[m]))*math.Abs(d[m+1])+safmin {
							break
						}
					}
				} else {
					m = lend
				}
				if m < lend {
					e[m] = 0
				}
				p := d[l]
				if m == l {
					// Eigenvalue found.
					l++
					if l > lend {
						break
					}
					continue
				}

				// If remaining matrix is 2×2, use Dlae2 to compute its eigensystem.
				if m == l+1 {
					if icompz > 0 {
						d[l], d[l+1], work[l], work[n-1+l] = impl.Dlaev2(d[l], e[l], d[l+1])
						impl.Dlasr(blas.Right, lapack.Variable, lapack.Backward,
							n, 2, work[l:], work[n-1+l:], z[l:], ldz)
					} else {
						d[l], d[l+1] = impl.Dlae2(d[l], e[l], d[l+1])
					}
					e[l] = 0
					l += 2
					if l > lend {
						break
					}
					continue
				}

				if jtot == nmaxit {
					break
				}
				jtot++

				// Form shift
				g := (d[l+1] - p) / (2 * e[l])
				r := impl.Dlapy2(g, 1)
				g = d[m] - p + e[l]/(g+math.Copysign(r, g))
				s := 1.0
				c := 1.0
				p = 0.0

				// Inner loop
				for i := m - 1; i >= l; i-- {
					f := s * e[i]
					b := c * e[i]
					c, s, r = impl.Dlartg(g, f)
					if i != m-1 {
						e[i+1] = r
					}
					g = d[i+1] - p
					r = (d[i]-g)*s + 2*c*b
					p = s * r
					d[i+1] = g + p
					g = c*r - b

					// If eigenvectors are desired, then save rotations.
					if icompz > 0 {
						work[i] = c
						work[n-1+i] = -s
					}
				}
				// If eigenvectors are desired, then apply saved rotations.
				if icompz > 0 {
					mm := m - l + 1
					impl.Dlasr(blas.Right, lapack.Variable, lapack.Backward,
						n, mm, work[l:], work[n-1+l:], z[l:], ldz)
				}
				d[l] -= p
				e[l] = g
			}
		} else {
			// QR Iteration.
			// Look for small superdiagonal element.
			for {
				if l != lend {
					for m = l; m > lend; m-- {
						v := math.Abs(e[m-1])
						if v*v <= (eps2*math.Abs(d[m])*math.Abs(d[m-1]) + safmin) {
							break
						}
					}
				} else {
					m = lend
				}
				if m > lend {
					e[m-1] = 0
				}
				p := d[l]
				if m == l {
					// Eigenvalue found
					l--
					if l < lend {
						break
					}
					continue
				}

				// If remaining matrix is 2×2, use Dlae2 to compute its eigenvalues.
				if m == l-1 {
					if icompz > 0 {
						d[l-1], d[l], work[m], work[n-1+m] = impl.Dlaev2(d[l-1], e[l-1], d[l])
						impl.Dlasr(blas.Right, lapack.Variable, lapack.Forward,
							n, 2, work[m:], work[n-1+m:], z[l-1:], ldz)
					} else {
						d[l-1], d[l] = impl.Dlae2(d[l-1], e[l-1], d[l])
					}
					e[l-1] = 0
					l -= 2
					if l < lend {
						break
					}
					continue
				}
				if jtot == nmaxit {
					break
				}
				jtot++

				// Form shift.
				g := (d[l-1] - p) / (2 * e[l-1])
				r := impl.Dlapy2(g, 1)
				g = d[m] - p + (e[l-1])/(g+math.Copysign(r, g))
				s := 1.0
				c := 1.0
				p = 0.0

				// Inner loop.
				for i := m; i < l; i++ {
					f := s * e[i]
					b := c * e[i]
					c, s, r = impl.Dlartg(g, f)
					if i != m {
						e[i-1] = r
					}
					g = d[i] - p
					r = (d[i+1]-g)*s + 2*c*b
					p = s * r
					d[i] = g + p
					g = c*r - b

					// If eigenvectors are desired, then save rotations.
					if icompz > 0 {
						work[i] = c
						work[n-1+i] = s
					}
				}

				// If eigenvectors are desired, then apply saved rotations.
				if icompz > 0 {
					mm := l - m + 1
					impl.Dlasr(blas.Right, lapack.Variable, lapack.Forward,
						n, mm, work[m:], work[n-1+m:], z[m:], ldz)
				}
				d[l] -= p
				e[l-1] = g
			}
		}

		// Undo scaling if necessary.
		switch iscale {
		case down:
			impl.Dlascl(lapack.General, 0, 0, ssfmax, anorm, lendsv-lsv+1, 1, d[lsv:], n)
			impl.Dlascl(lapack.General, 0, 0, ssfmax, anorm, lendsv-lsv, 1, e[lsv:], n)
		case up:
			impl.Dlascl(lapack.General, 0, 0, ssfmin, anorm, lendsv-lsv+1, 1, d[lsv:], n)
			impl.Dlascl(lapack.General, 0, 0, ssfmin, anorm, lendsv-lsv, 1, e[lsv:], n)
		}

		// Check for no convergence to an eigenvalue after a total of n*maxit iterations.
		if jtot >= nmaxit {
			break
		}
	}
	for i := 0; i < n-1; i++ {
		if e[i] != 0 {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package paxos

import (
	"borg/assert"
	"testing"
)

type Putter interface {
	Put(m Msg)
}

type Instance struct {
	vin  chan string
	vout chan string

	// Coordinator
	cIns chan Msg

	// Acceptor
	aIns chan Msg

	// Learner
	lIns  chan Msg
}

func (ins *Instance) Put(m Msg) {
	go func() { ins.cIns <- m }()
	go func() { ins.aIns <- m }()
	go func() { ins.lIns <- m }()
}

func (ins *Instance) Value() string {
	return <-ins.vout
}

func NewInstance() *Instance {
	return &Instance{
		vin: make(chan string),
		vout: make(chan string),
		cIns:  make(chan Msg),
		aIns:  make(chan Msg),
		lIns:  make(chan Msg),
	}
}

func (ins *Instance) Init(p Putter) {
	msgs := make(chan Msg)
	go coordinator(1, 1, 3, ins.vin, ins.cIns, msgs, make(chan int))
	go acceptor(2, ins.aIns, msgs)
	go learner(1, ins.lIns, ins.vout, func() {})
	go func() {
		for m := range msgs {
			p.Put(m)
		}
	}()
}

func (ins *Instance) Propose(v string) {
	ins.vin <- v
}

// Testing

func TestStartAtLearn(t *testing.T) {
	ins := NewInstance()
	ins.Init(ins)
	ins.Put(m("1:*:VOTE:1:foo"))
	ins.Put(m("1:*:VOTE:1:foo"))
	ins.Put(m("1:*:VOTE:1:foo"))
	assert.Equal(t, "foo", ins.Value(), "")
}

func TestStartAtAccept(t *testing.T) {
	ins := NewInstance()
	ins.Init(ins)
	ins.Put(m("1:*:NOMINATE:1:foo"))
	ins.Put(m("1:*:NOMINATE:1:foo"))
	ins.Put(m("1:*:NOMINATE:1:foo"))
	assert.Equal(t, "foo", ins.Value(), "")
}

func TestStartAtCoord(t *testing.T) {
	ins := NewInstance()
	ins.Init(ins)
	ins.Propose("foo")
	assert.Equal(t, "foo", ins.Value(), "")
}
<commit_msg>paxos: test quorum > 1<commit_after>package paxos

import (
	"borg/assert"
	"testing"
)

type Putter interface {
	Put(m Msg)
}

type Instance struct {
	quorum uint64

	vin  chan string
	vout chan string

	// Coordinator
	cIns chan Msg

	// Acceptor
	aIns chan Msg

	// Learner
	lIns  chan Msg
}

func (ins *Instance) Put(m Msg) {
	go func() { ins.cIns <- m }()
	go func() { ins.aIns <- m }()
	go func() { ins.lIns <- m }()
}

func (ins *Instance) Value() string {
	return <-ins.vout
}

func NewInstance(quorum uint64) *Instance {
	return &Instance{
		quorum: quorum,
		vin: make(chan string),
		vout: make(chan string),
		cIns:  make(chan Msg),
		aIns:  make(chan Msg),
		lIns:  make(chan Msg),
	}
}

func (ins *Instance) Init(p Putter) {
	msgs := make(chan Msg)
	go coordinator(1, ins.quorum, 3, ins.vin, ins.cIns, msgs, make(chan int))
	go acceptor(2, ins.aIns, msgs)
	go learner(1, ins.lIns, ins.vout, func() {})
	go func() {
		for m := range msgs {
			p.Put(m)
		}
	}()
}

func (ins *Instance) Propose(v string) {
	ins.vin <- v
}

// Testing

func TestStartAtLearn(t *testing.T) {
	ins := NewInstance(1)
	ins.Init(ins)
	ins.Put(m("1:*:VOTE:1:foo"))
	ins.Put(m("1:*:VOTE:1:foo"))
	ins.Put(m("1:*:VOTE:1:foo"))
	assert.Equal(t, "foo", ins.Value(), "")
}

func TestStartAtAccept(t *testing.T) {
	ins := NewInstance(1)
	ins.Init(ins)
	ins.Put(m("1:*:NOMINATE:1:foo"))
	ins.Put(m("1:*:NOMINATE:1:foo"))
	ins.Put(m("1:*:NOMINATE:1:foo"))
	assert.Equal(t, "foo", ins.Value(), "")
}

func TestStartAtCoord(t *testing.T) {
	ins := NewInstance(1)
	ins.Init(ins)
	ins.Propose("foo")
	assert.Equal(t, "foo", ins.Value(), "")
}

type FakePutter []Putter

func (fp FakePutter) Put(m Msg) {
	for _, p := range fp {
		p.Put(m)
	}
}

func TestMultipleInstances(t *testing.T) {
	insA := NewInstance(2)
	insB := NewInstance(2)
	insC := NewInstance(2)
	ps := []Putter{insA, insB, insC}
	insA.Init(FakePutter(ps))
	insB.Init(FakePutter(ps))
	insC.Init(FakePutter(ps))

	insA.Propose("bar")
	assert.Equal(t, "bar", insA.Value(), "")
}
<|endoftext|>


<commit_before>package db

import (
	"container/list"
	"fmt"

	turtle "github.com/gtfierro/hod/goraptor"
	"github.com/gtfierro/hod/query"
)

var emptyTree = newPointerTree(3)

// queryContext
type queryContext struct {
	candidates       map[string]*pointerTree
	chains           map[Key]map[string]*linkRecord
	db               *DB
	traverseOrder    *list.List
	traverseVars     map[string]*list.Element
	linkedValueCache map[Key]map[string]*pointerTree
	tupleCache       map[string][]map[string]turtle.URI
	// embedded query plan
	*queryPlan
}

func newQueryContext(plan *queryPlan, db *DB) *queryContext {
	ctx := &queryContext{
		candidates:       make(map[string]*pointerTree),
		chains:           make(map[Key]map[string]*linkRecord),
		queryPlan:        plan,
		traverseOrder:    list.New(),
		traverseVars:     make(map[string]*list.Element),
		linkedValueCache: make(map[Key]map[string]*pointerTree),
		tupleCache:       make(map[string][]map[string]turtle.URI),
		db:               db,
	}
	return ctx
}

func (ctx *queryContext) dumpVarCounts() {
	for varname, tree := range ctx.candidates {
		fmt.Println("var count", varname, tree.Len())
	}
}

func (ctx *queryContext) dumpTraverseOrder() {
	elem := ctx.traverseOrder.Front()
	for elem.Next() != nil {
		varname := elem.Value.(string)
		fmt.Println(varname, "next =>", elem.Next().Value.(string))
		elem = elem.Next()
	}
	fmt.Println(elem.Value.(string))
}

// now we need to plan out the set of actions for adding/filtering vars on the query context

// returns the set of current guesses for the given variable
// returns TRUE if the tree is known, FALSE otherwise
func (ctx *queryContext) getValues(varname string) (*pointerTree, bool) {
	if tree, found := ctx.candidates[varname]; found && tree != nil {
		return tree, true
	}
	return emptyTree, false
}

func (ctx *queryContext) candidateHasValue(varname string, ent *Entity) bool {
	tree, found := ctx.candidates[varname]
	if found {
		return tree.Has(ent)
	}
	return true // no values to filter, so we just say "yes"
}

// returns the set of reachable values from the given entity
func (ctx *queryContext) getLinkedValues(varname string, ent *Entity) *pointerTree {
	if tree, found := ctx.linkedValueCache[ent.PK][varname]; found {
		return tree
	}
	var res = newPointerTree(3)
	chain := ctx.chains[ent.PK][varname]
	if chain != nil {
		for _, link := range chain.links {
			res.Add(ctx.db.MustGetEntityFromHash(link.me))
		}
	}
	if _, found := ctx.linkedValueCache[ent.PK]; !found {
		ctx.linkedValueCache[ent.PK] = make(map[string]*pointerTree)
	}
	ctx.linkedValueCache[ent.PK][varname] = res
	return res
}

// if we already have values for the given variable name, we filter the values given by those
// (take the intersection). Else we just keep the provided values the same
func (ctx *queryContext) filterIfDefined(varname string, values *pointerTree) *pointerTree {
	if tree, found := ctx.getValues(varname); found {
		values = intersectPointerTrees(tree, values)
	}
	return values
}

func (ctx *queryContext) define(varname string, values *pointerTree) {
	ctx.candidates[varname] = values
	_, found := ctx.traverseVars[varname]
	if !found {
		elem := ctx.traverseOrder.PushBack(varname)
		ctx.traverseVars[varname] = elem
	}
}

// if values don't exist for the variable w/n this context, then we just add these values
// if values DO already exist, then we take the intersection
func (ctx *queryContext) addOrFilterVariable(varname string, values *pointerTree) {
	if oldValues, exists := ctx.candidates[varname]; exists {
		ctx.candidates[varname] = intersectPointerTrees(oldValues, values)
		values = ctx.candidates[varname]
	} else {
		ctx.candidates[varname] = values
	}

	_, found := ctx.traverseVars[varname]
	if !found {
		elem := ctx.traverseOrder.PushBack(varname)
		ctx.traverseVars[varname] = elem
	}

}

// unions, not intersects
func (ctx *queryContext) addOrMergeVariable(varname string, values *pointerTree) {
	if oldValues, exists := ctx.candidates[varname]; exists {
		mergePointerTrees(oldValues, values)
		ctx.candidates[varname] = oldValues
	} else {
		ctx.candidates[varname] = values
	}

	_, found := ctx.traverseVars[varname]
	if !found {
		elem := ctx.traverseOrder.PushBack(varname)
		ctx.traverseVars[varname] = elem
	}
}
func (ctx *queryContext) addReachable(parent *Entity, parentVar string, reachable *pointerTree, reachableVar string) {
	chain, found := ctx.chains[parent.PK][reachableVar]
	if !found {
		chain = &linkRecord{me: parent.PK}
	}
	reachable.mergeOntoLinkRecord(chain)
	if _, found := ctx.chains[parent.PK]; !found {
		ctx.chains[parent.PK] = make(map[string]*linkRecord)
	}
	ctx.chains[parent.PK][reachableVar] = chain

	parentElem, found := ctx.traverseVars[parentVar]
	if !found {
		parentElem = ctx.traverseOrder.PushBack(parentVar)
		ctx.traverseVars[parentVar] = parentElem
	}

	childElem, found := ctx.traverseVars[reachableVar]
	if found {
		ctx.traverseOrder.MoveAfter(childElem, parentElem)
	} else {
		elem := ctx.traverseOrder.InsertAfter(reachableVar, parentElem)
		ctx.traverseVars[reachableVar] = elem
	}
}

func (ctx *queryContext) addReachableSingle(parent *Entity, parentVar string, reachable *Entity, reachableVar string) {
	chain, found := ctx.chains[parent.PK][reachableVar]
	if !found {
		chain = &linkRecord{me: parent.PK}
	}
	// add on this one record
	chain.links = append(chain.links, &linkRecord{me: reachable.PK})
	if _, found := ctx.chains[parent.PK]; !found {
		ctx.chains[parent.PK] = make(map[string]*linkRecord)
	}
	ctx.chains[parent.PK][reachableVar] = chain

	parentElem, found := ctx.traverseVars[parentVar]
	if !found {
		parentElem = ctx.traverseOrder.PushBack(parentVar)
		ctx.traverseVars[parentVar] = parentElem
	}

	childElem, found := ctx.traverseVars[reachableVar]
	if found {
		ctx.traverseOrder.MoveAfter(childElem, parentElem)
	} else {
		elem := ctx.traverseOrder.InsertAfter(reachableVar, parentElem)
		ctx.traverseVars[reachableVar] = elem
	}
}

// returns true if any vars are reachable from this entity
func (ctx *queryContext) entityHasFollowers(ent *Entity) bool {
	if ent == nil || ent.PK == emptyHash {
		return false
	}
	if m, found := ctx.chains[ent.PK]; found && len(m) > 0 {
		for _, links := range m {
			if len(links.links) > 0 {
				return true
			}
		}
	}
	return false
}

func (ctx *queryContext) expandTuples() [][]turtle.URI {
	var (
		startvar string
		results  [][]turtle.URI
		tuples   []map[string]turtle.URI
	)
	// choose first variable
	for v, state := range ctx.vars {
		if state == RESOLVED {
			startvar = v
			break
		}
	}
	if len(startvar) == 0 {
		// need to choose the "parent" if there is no RESOLVED variable
		for _, parent := range ctx.vars {
			if _, exists := ctx.vars[parent]; !exists {
				startvar = parent
				break
			}
		}
	}

	topVarTree := ctx.candidates[startvar]
	if topVarTree == nil {
		return results // fail early
	}
	max := topVarTree.Max()
	iter := func(ent *Entity) bool {
		newtups := ctx._getTuplesFromTree(startvar, ent)
		tuples = append(tuples, newtups...)
		return ent != max
	}
	topVarTree.Iter(iter)
tupleLoop:
	for _, tup := range tuples {
		var row []turtle.URI
		for _, varname := range ctx.selectVars {
			if _, found := tup[varname]; !found {
				if ctx.query.Select.Partial {
					continue
				} else {
					continue tupleLoop
				}
			}
			row = append(row, tup[varname])
		}
		results = append(results, row)
		if ctx.query.Select.Limit > 0 && len(results) == ctx.query.Select.Limit {
			return results
		}
	}
	return results
}

func (ctx *queryContext) _getTuplesFromTree(name string, ent *Entity) []map[string]turtle.URI {
	var ret []map[string]turtle.URI
	if ent == nil || ent.PK == emptyHash {
		return ret
	}
	uri := ctx.db.MustGetURI(ent.PK)
	if ret, found := ctx.tupleCache[name+ent.PK.String()]; found {
		return ret
	}

	vars := make(map[string]turtle.URI)
	vars[name] = uri
	if !ctx.entityHasFollowers(ent) {
		ret = append(ret, map[string]turtle.URI{name: uri})
	} else {
		// loop through the values of the child var
		for childName, _links := range ctx.chains[ent.PK] {
			if len(_links.links) == 0 {
				continue
			}
			if childName == name {
				log.Debug("continuing from", uri, childName)
				log.Debug(ret)
				ret = append(ret, map[string]turtle.URI{name: uri})
				continue
			}
			childValues := ctx.getLinkedValues(childName, ent)
			max := childValues.Max()
			iter := func(child *Entity) bool {
				for _, m := range ctx._getTuplesFromTree(childName, child) {
					for k, v := range m {
						vars[k] = v
					}
					// when we want to append, make sure to allocate a new map
					newvar := make(map[string]turtle.URI)
					for k, v := range vars {
						newvar[k] = v
					}
					ret = append(ret, newvar)
				}
				return child != max
			}
			childValues.Iter(iter)
		}
	}
	ctx.tupleCache[name+ent.PK.String()] = ret
	return ret
}

const (
	RESOLVED   = "RESOLVED"
	UNRESOLVED = ""
)

// contains all useful state information for executing a query
type queryPlan struct {
	operations []operation
	selectVars []string
	dg         *dependencyGraph
	query      query.Query
	vars       map[string]string
}

func newQueryPlan(dg *dependencyGraph, q query.Query) *queryPlan {
	plan := &queryPlan{
		selectVars: dg.selectVars,
		dg:         dg,
		query:      q,
		vars:       make(map[string]string),
	}
	return plan
}

func (qp *queryPlan) dumpVarchain() {
	for k, v := range qp.vars {
		fmt.Println(k, "=>", v)
	}
}

func (plan *queryPlan) hasVar(variable string) bool {
	return plan.vars[variable] != UNRESOLVED
}

func (plan *queryPlan) varIsChild(variable string) bool {
	return plan.hasVar(variable) && plan.vars[variable] != RESOLVED
}

func (plan *queryPlan) varIsTop(variable string) bool {
	return plan.hasVar(variable) && plan.vars[variable] == RESOLVED
}

func (plan *queryPlan) addTopLevel(variable string) {
	plan.vars[variable] = RESOLVED
}

func (plan *queryPlan) addLink(parent, child string) {
	plan.vars[child] = parent
}
<commit_msg>remove debug<commit_after>package db

import (
	"container/list"
	"fmt"

	turtle "github.com/gtfierro/hod/goraptor"
	"github.com/gtfierro/hod/query"
)

var emptyTree = newPointerTree(3)

// queryContext
type queryContext struct {
	candidates       map[string]*pointerTree
	chains           map[Key]map[string]*linkRecord
	db               *DB
	traverseOrder    *list.List
	traverseVars     map[string]*list.Element
	linkedValueCache map[Key]map[string]*pointerTree
	tupleCache       map[string][]map[string]turtle.URI
	// embedded query plan
	*queryPlan
}

func newQueryContext(plan *queryPlan, db *DB) *queryContext {
	ctx := &queryContext{
		candidates:       make(map[string]*pointerTree),
		chains:           make(map[Key]map[string]*linkRecord),
		queryPlan:        plan,
		traverseOrder:    list.New(),
		traverseVars:     make(map[string]*list.Element),
		linkedValueCache: make(map[Key]map[string]*pointerTree),
		tupleCache:       make(map[string][]map[string]turtle.URI),
		db:               db,
	}
	return ctx
}

func (ctx *queryContext) dumpVarCounts() {
	for varname, tree := range ctx.candidates {
		fmt.Println("var count", varname, tree.Len())
	}
}

func (ctx *queryContext) dumpTraverseOrder() {
	elem := ctx.traverseOrder.Front()
	for elem.Next() != nil {
		varname := elem.Value.(string)
		fmt.Println(varname, "next =>", elem.Next().Value.(string))
		elem = elem.Next()
	}
	fmt.Println(elem.Value.(string))
}

// now we need to plan out the set of actions for adding/filtering vars on the query context

// returns the set of current guesses for the given variable
// returns TRUE if the tree is known, FALSE otherwise
func (ctx *queryContext) getValues(varname string) (*pointerTree, bool) {
	if tree, found := ctx.candidates[varname]; found && tree != nil {
		return tree, true
	}
	return emptyTree, false
}

func (ctx *queryContext) candidateHasValue(varname string, ent *Entity) bool {
	tree, found := ctx.candidates[varname]
	if found {
		return tree.Has(ent)
	}
	return true // no values to filter, so we just say "yes"
}

// returns the set of reachable values from the given entity
func (ctx *queryContext) getLinkedValues(varname string, ent *Entity) *pointerTree {
	if tree, found := ctx.linkedValueCache[ent.PK][varname]; found {
		return tree
	}
	var res = newPointerTree(3)
	chain := ctx.chains[ent.PK][varname]
	if chain != nil {
		for _, link := range chain.links {
			res.Add(ctx.db.MustGetEntityFromHash(link.me))
		}
	}
	if _, found := ctx.linkedValueCache[ent.PK]; !found {
		ctx.linkedValueCache[ent.PK] = make(map[string]*pointerTree)
	}
	ctx.linkedValueCache[ent.PK][varname] = res
	return res
}

// if we already have values for the given variable name, we filter the values given by those
// (take the intersection). Else we just keep the provided values the same
func (ctx *queryContext) filterIfDefined(varname string, values *pointerTree) *pointerTree {
	if tree, found := ctx.getValues(varname); found {
		values = intersectPointerTrees(tree, values)
	}
	return values
}

func (ctx *queryContext) define(varname string, values *pointerTree) {
	ctx.candidates[varname] = values
	_, found := ctx.traverseVars[varname]
	if !found {
		elem := ctx.traverseOrder.PushBack(varname)
		ctx.traverseVars[varname] = elem
	}
}

// if values don't exist for the variable w/n this context, then we just add these values
// if values DO already exist, then we take the intersection
func (ctx *queryContext) addOrFilterVariable(varname string, values *pointerTree) {
	if oldValues, exists := ctx.candidates[varname]; exists {
		ctx.candidates[varname] = intersectPointerTrees(oldValues, values)
		values = ctx.candidates[varname]
	} else {
		ctx.candidates[varname] = values
	}

	_, found := ctx.traverseVars[varname]
	if !found {
		elem := ctx.traverseOrder.PushBack(varname)
		ctx.traverseVars[varname] = elem
	}

}

// unions, not intersects
func (ctx *queryContext) addOrMergeVariable(varname string, values *pointerTree) {
	if oldValues, exists := ctx.candidates[varname]; exists {
		mergePointerTrees(oldValues, values)
		ctx.candidates[varname] = oldValues
	} else {
		ctx.candidates[varname] = values
	}

	_, found := ctx.traverseVars[varname]
	if !found {
		elem := ctx.traverseOrder.PushBack(varname)
		ctx.traverseVars[varname] = elem
	}
}
func (ctx *queryContext) addReachable(parent *Entity, parentVar string, reachable *pointerTree, reachableVar string) {
	chain, found := ctx.chains[parent.PK][reachableVar]
	if !found {
		chain = &linkRecord{me: parent.PK}
	}
	reachable.mergeOntoLinkRecord(chain)
	if _, found := ctx.chains[parent.PK]; !found {
		ctx.chains[parent.PK] = make(map[string]*linkRecord)
	}
	ctx.chains[parent.PK][reachableVar] = chain

	parentElem, found := ctx.traverseVars[parentVar]
	if !found {
		parentElem = ctx.traverseOrder.PushBack(parentVar)
		ctx.traverseVars[parentVar] = parentElem
	}

	childElem, found := ctx.traverseVars[reachableVar]
	if found {
		ctx.traverseOrder.MoveAfter(childElem, parentElem)
	} else {
		elem := ctx.traverseOrder.InsertAfter(reachableVar, parentElem)
		ctx.traverseVars[reachableVar] = elem
	}
}

func (ctx *queryContext) addReachableSingle(parent *Entity, parentVar string, reachable *Entity, reachableVar string) {
	chain, found := ctx.chains[parent.PK][reachableVar]
	if !found {
		chain = &linkRecord{me: parent.PK}
	}
	// add on this one record
	chain.links = append(chain.links, &linkRecord{me: reachable.PK})
	if _, found := ctx.chains[parent.PK]; !found {
		ctx.chains[parent.PK] = make(map[string]*linkRecord)
	}
	ctx.chains[parent.PK][reachableVar] = chain

	parentElem, found := ctx.traverseVars[parentVar]
	if !found {
		parentElem = ctx.traverseOrder.PushBack(parentVar)
		ctx.traverseVars[parentVar] = parentElem
	}

	childElem, found := ctx.traverseVars[reachableVar]
	if found {
		ctx.traverseOrder.MoveAfter(childElem, parentElem)
	} else {
		elem := ctx.traverseOrder.InsertAfter(reachableVar, parentElem)
		ctx.traverseVars[reachableVar] = elem
	}
}

// returns true if any vars are reachable from this entity
func (ctx *queryContext) entityHasFollowers(ent *Entity) bool {
	if ent == nil || ent.PK == emptyHash {
		return false
	}
	if m, found := ctx.chains[ent.PK]; found && len(m) > 0 {
		for _, links := range m {
			if len(links.links) > 0 {
				return true
			}
		}
	}
	return false
}

func (ctx *queryContext) expandTuples() [][]turtle.URI {
	var (
		startvar string
		results  [][]turtle.URI
		tuples   []map[string]turtle.URI
	)
	// choose first variable
	for v, state := range ctx.vars {
		if state == RESOLVED {
			startvar = v
			break
		}
	}
	if len(startvar) == 0 {
		// need to choose the "parent" if there is no RESOLVED variable
		for _, parent := range ctx.vars {
			if _, exists := ctx.vars[parent]; !exists {
				startvar = parent
				break
			}
		}
	}

	topVarTree := ctx.candidates[startvar]
	if topVarTree == nil {
		return results // fail early
	}
	max := topVarTree.Max()
	iter := func(ent *Entity) bool {
		newtups := ctx._getTuplesFromTree(startvar, ent)
		tuples = append(tuples, newtups...)
		return ent != max
	}
	topVarTree.Iter(iter)
tupleLoop:
	for _, tup := range tuples {
		var row []turtle.URI
		for _, varname := range ctx.selectVars {
			if _, found := tup[varname]; !found {
				if ctx.query.Select.Partial {
					continue
				} else {
					continue tupleLoop
				}
			}
			row = append(row, tup[varname])
		}
		results = append(results, row)
		if ctx.query.Select.Limit > 0 && len(results) == ctx.query.Select.Limit {
			return results
		}
	}
	return results
}

func (ctx *queryContext) _getTuplesFromTree(name string, ent *Entity) []map[string]turtle.URI {
	var ret []map[string]turtle.URI
	if ent == nil || ent.PK == emptyHash {
		return ret
	}
	uri := ctx.db.MustGetURI(ent.PK)
	if ret, found := ctx.tupleCache[name+ent.PK.String()]; found {
		return ret
	}

	vars := make(map[string]turtle.URI)
	vars[name] = uri
	if !ctx.entityHasFollowers(ent) {
		ret = append(ret, map[string]turtle.URI{name: uri})
	} else {
		// loop through the values of the child var
		for childName, _links := range ctx.chains[ent.PK] {
			if len(_links.links) == 0 {
				continue
			}
			if childName == name {
				ret = append(ret, map[string]turtle.URI{name: uri})
				continue
			}
			childValues := ctx.getLinkedValues(childName, ent)
			max := childValues.Max()
			iter := func(child *Entity) bool {
				for _, m := range ctx._getTuplesFromTree(childName, child) {
					for k, v := range m {
						vars[k] = v
					}
					// when we want to append, make sure to allocate a new map
					newvar := make(map[string]turtle.URI)
					for k, v := range vars {
						newvar[k] = v
					}
					ret = append(ret, newvar)
				}
				return child != max
			}
			childValues.Iter(iter)
		}
	}
	ctx.tupleCache[name+ent.PK.String()] = ret
	return ret
}

const (
	RESOLVED   = "RESOLVED"
	UNRESOLVED = ""
)

// contains all useful state information for executing a query
type queryPlan struct {
	operations []operation
	selectVars []string
	dg         *dependencyGraph
	query      query.Query
	vars       map[string]string
}

func newQueryPlan(dg *dependencyGraph, q query.Query) *queryPlan {
	plan := &queryPlan{
		selectVars: dg.selectVars,
		dg:         dg,
		query:      q,
		vars:       make(map[string]string),
	}
	return plan
}

func (qp *queryPlan) dumpVarchain() {
	for k, v := range qp.vars {
		fmt.Println(k, "=>", v)
	}
}

func (plan *queryPlan) hasVar(variable string) bool {
	return plan.vars[variable] != UNRESOLVED
}

func (plan *queryPlan) varIsChild(variable string) bool {
	return plan.hasVar(variable) && plan.vars[variable] != RESOLVED
}

func (plan *queryPlan) varIsTop(variable string) bool {
	return plan.hasVar(variable) && plan.vars[variable] == RESOLVED
}

func (plan *queryPlan) addTopLevel(variable string) {
	plan.vars[variable] = RESOLVED
}

func (plan *queryPlan) addLink(parent, child string) {
	plan.vars[child] = parent
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package provider

import (
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/metrics/pkg/apis/custom_metrics"
)

// MetricInfo describes a metric for a particular
// fully-qualified group resource.
type MetricInfo struct {
	GroupResource          schema.GroupResource
	Namespaced             bool
	Metric                 string
}

// CustomMetricsProvider is a soruce of custom metrics
// which is able to supply a list of available metrics,
// as well as metric values themselves on demand.
//
// Note that group-resources are provided  as GroupResources,
// not GroupKinds.  This is to allow flexibility on the part
// of the implementor: implementors do not necessarily need
// to be aware of all existing kinds and their corresponding
// REST mappings in order to perform queries.
//
// For queries that use label selectors, it is up to the
// implementor to decide how to make use of the label selector --
// they may wish to query the main Kubernetes API server, or may
// wish to simply make use of stored information in their TSDB.
type CustomMetricsProvider interface {
	// GetRootScopedMetricByName fetches a particular metric for a particular root-scoped object.
	GetRootScopedMetricByName(groupResource schema.GroupResource, name string, metricName string) (*custom_metrics.MetricValue, error)

	// GetRootScopedMetricByName fetches a particular metric for a set of root-scoped objects
	// matching the given label selector.
	GetRootScopedMetricBySelector(groupResource schema.GroupResource, selector labels.Selector, metricName string) (*custom_metrics.MetricValueList, error)

	// GetNamespacedMetricByName fetches a particular metric for a particular namespaced object.
	GetNamespacedMetricByName(groupResource schema.GroupResource, namespace string, name string, metricName string) (*custom_metrics.MetricValue, error)

	// GetNamespacedMetricByName fetches a particular metric for a set of namespaced objects
	// matching the given label selector.
	GetNamespacedMetricBySelector(groupResource schema.GroupResource, namespace string, selector labels.Selector, metricName string) (*custom_metrics.MetricValueList, error)

	// ListAllMetrics provides a list of all available metrics at
	// the current time.  Note that this is not allowed to return
	// an error, so it is reccomended that implementors cache and
	// periodically update this list, instead of querying every time.
	ListAllMetrics() []MetricInfo
}
<commit_msg>Add String() implementation for MetricInfo<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package provider

import (
	"fmt"

	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/metrics/pkg/apis/custom_metrics"
)

// MetricInfo describes a metric for a particular
// fully-qualified group resource.
type MetricInfo struct {
	GroupResource schema.GroupResource
	Namespaced    bool
	Metric        string
}

func (i MetricInfo) String() string {
	if i.Namespaced {
		return fmt.Sprintf("%s/%s(namespaced)", i.GroupResource.String(), i.Metric)
	} else {
		return fmt.Sprintf("%s/%s", i.GroupResource.String(), i.Metric)
	}
}

// CustomMetricsProvider is a soruce of custom metrics
// which is able to supply a list of available metrics,
// as well as metric values themselves on demand.
//
// Note that group-resources are provided  as GroupResources,
// not GroupKinds.  This is to allow flexibility on the part
// of the implementor: implementors do not necessarily need
// to be aware of all existing kinds and their corresponding
// REST mappings in order to perform queries.
//
// For queries that use label selectors, it is up to the
// implementor to decide how to make use of the label selector --
// they may wish to query the main Kubernetes API server, or may
// wish to simply make use of stored information in their TSDB.
type CustomMetricsProvider interface {
	// GetRootScopedMetricByName fetches a particular metric for a particular root-scoped object.
	GetRootScopedMetricByName(groupResource schema.GroupResource, name string, metricName string) (*custom_metrics.MetricValue, error)

	// GetRootScopedMetricByName fetches a particular metric for a set of root-scoped objects
	// matching the given label selector.
	GetRootScopedMetricBySelector(groupResource schema.GroupResource, selector labels.Selector, metricName string) (*custom_metrics.MetricValueList, error)

	// GetNamespacedMetricByName fetches a particular metric for a particular namespaced object.
	GetNamespacedMetricByName(groupResource schema.GroupResource, namespace string, name string, metricName string) (*custom_metrics.MetricValue, error)

	// GetNamespacedMetricByName fetches a particular metric for a set of namespaced objects
	// matching the given label selector.
	GetNamespacedMetricBySelector(groupResource schema.GroupResource, namespace string, selector labels.Selector, metricName string) (*custom_metrics.MetricValueList, error)

	// ListAllMetrics provides a list of all available metrics at
	// the current time.  Note that this is not allowed to return
	// an error, so it is reccomended that implementors cache and
	// periodically update this list, instead of querying every time.
	ListAllMetrics() []MetricInfo
}
<|endoftext|>


<commit_before>package store

import (
	"bytes"
	"fmt"
	"reflect"
	"sort"
	"strings"
	"sync"

	"github.com/Aptomi/aptomi/pkg/runtime"
)

var (
	indexCacheMu = &sync.Mutex{}
	indexCache   = map[runtime.Kind]*Indexes{}
)

const LastGenIndex = ""

type Indexes struct {
	List map[string]*Index
}

func (indexes *Indexes) NameForStorable(indexName string, storable runtime.Storable, codec Codec) string {
	if index, exist := indexes.List[indexName]; exist {
		return index.NameForStorable(storable, codec)
	} else {
		panic(fmt.Sprintf("trying to access non-existing indexName for kind %s: %s", storable.GetKind(), indexName))
	}
}

func (indexes *Indexes) NameForValue(indexName string, key runtime.Key, value interface{}, codec Codec) string {
	if index, exist := indexes.List[indexName]; exist {
		return index.NameForValue(key, value, codec)
	} else {
		panic(fmt.Sprintf("trying to access non-existing indexName for key %s: %s", key, indexName))
	}
}

func IndexesFor(info *runtime.TypeInfo) *Indexes {
	indexCacheMu.Lock()
	defer indexCacheMu.Unlock()

	indexes, exist := indexCache[info.Kind]
	if !exist {
		indexes = &Indexes{map[string]*Index{}}
		indexCache[info.Kind] = indexes

		if info.Versioned {
			indexes.List[LastGenIndex] = &Index{
				Type: IndexTypeLastGen,
			}
		}

		t := reflect.TypeOf(info.New())
		if t.Kind() == reflect.Ptr {
			t = t.Elem()
		}
		for i := 0; i < t.NumField(); i++ {
			f := t.Field(i)
			tag := f.Tag.Get("store")

			if strings.Contains(tag, "index") {
				// todo validate that field is accessible
				transformer := info.IndexValueTransforms[f.Name]
				if transformer == nil {
					transformer = func(val interface{}) interface{} {
						return val
					}
				}
				indexes.List[f.Name] = &Index{
					Type:           IndexTypeListGen,
					Field:          f.Name,
					ValueTransform: transformer,
					rFieldId:       i,
				}
			}
		}
	}

	return indexes
}

type IndexType int

const (
	IndexTypeUndef IndexType = iota
	IndexTypeLastGen
	IndexTypeListGen
)

func (indexType IndexType) String() string {
	indexTypes := [...]string{
		"lastgen",
		"listgen",
	}

	if indexType < 1 || indexType > 2 {
		panic(fmt.Sprintf("unknown index type: %d", indexType))
	}

	return indexTypes[indexType-1]
}

type Index struct {
	Type           IndexType
	Field          string
	ValueTransform runtime.ValueTransform
	rFieldId       int
}

func (index *Index) NameForStorable(storable runtime.Storable, codec Codec) string {
	key := runtime.KeyForStorable(storable)

	if index.Type == IndexTypeLastGen {
		return index.NameForValue(key, nil, codec)
	}

	t := reflect.ValueOf(storable)
	if t.Kind() == reflect.Ptr {
		t = t.Elem()
	}
	f := t.Field(index.rFieldId)

	return index.NameForValue(key, f.Interface(), codec)
}

func (index *Index) NameForValue(key runtime.Key, value interface{}, codec Codec) string {
	value = index.ValueTransform(value)
	if value == nil {
		return ""
	}

	key = index.Type.String() + "/" + key
	if index.Type == IndexTypeLastGen {
		return key
	}

	key += "/" + index.Field + "="

	if valueStr, ok := value.(string); ok {
		return key + valueStr
	}

	if valueGen, ok := value.(runtime.Generation); ok {
		return key + valueGen.String()
	}

	data, err := codec.Marshal(value)
	if err != nil {
		panic(fmt.Sprintf("error marshalling index value %s=%v", index.Field, value))
	}

	return key + string(data)
}

type IndexValueList [][]byte

func (list *IndexValueList) Add(value []byte) {
	// binary search to get desired value index in the list
	valueIndex := sort.Search(len(*list), func(index int) bool {
		return bytes.Compare((*list)[index], value) >= 0
	})

	// value already present in the list
	if valueIndex < len(*list) && bytes.Equal((*list)[valueIndex], value) {
		return
	}

	// insert value into desired position
	*list = append(*list, nil)
	copy((*list)[valueIndex+1:], (*list)[valueIndex:])
	(*list)[valueIndex] = value
}

func (list *IndexValueList) Remove(value []byte) {
	// binary search to get value index in the list
	valueIndex := sort.Search(len(*list), func(index int) bool {
		return bytes.Compare((*list)[index], value) >= 0
	})

	// remove value from the list if exists
	if valueIndex < len(*list) {
		copy((*list)[valueIndex:], (*list)[valueIndex+1:])
		(*list)[len(*list)-1] = nil
		*list = (*list)[:len(*list)-1]
	}
}

func (list *IndexValueList) Contains(value []byte) bool {
	// binary search to get value index in the list
	valueIndex := sort.Search(len(*list), func(index int) bool {
		return bytes.Compare((*list)[index], value) >= 0
	})

	return valueIndex < len(*list) && bytes.Equal((*list)[valueIndex], value)
}

func (list *IndexValueList) First() []byte {
	if len(*list) == 0 {
		return nil
	}

	return (*list)[0]
}

func (list *IndexValueList) Last() []byte {
	if len(*list) == 0 {
		return nil
	}

	return (*list)[len(*list)-1]
}
<commit_msg>Correctly handle nil values in getting index name for value<commit_after>package store

import (
	"bytes"
	"fmt"
	"reflect"
	"sort"
	"strings"
	"sync"

	"github.com/Aptomi/aptomi/pkg/runtime"
)

var (
	indexCacheMu = &sync.Mutex{}
	indexCache   = map[runtime.Kind]*Indexes{}
)

const LastGenIndex = ""

type Indexes struct {
	List map[string]*Index
}

func (indexes *Indexes) NameForStorable(indexName string, storable runtime.Storable, codec Codec) string {
	if index, exist := indexes.List[indexName]; exist {
		return index.NameForStorable(storable, codec)
	} else {
		panic(fmt.Sprintf("trying to access non-existing indexName for kind %s: %s", storable.GetKind(), indexName))
	}
}

func (indexes *Indexes) NameForValue(indexName string, key runtime.Key, value interface{}, codec Codec) string {
	if index, exist := indexes.List[indexName]; exist {
		return index.NameForValue(key, value, codec)
	} else {
		panic(fmt.Sprintf("trying to access non-existing indexName for key %s: %s", key, indexName))
	}
}

var noopValueTransform = func(val interface{}) interface{} {
	return val
}

func IndexesFor(info *runtime.TypeInfo) *Indexes {
	indexCacheMu.Lock()
	defer indexCacheMu.Unlock()

	indexes, exist := indexCache[info.Kind]
	if !exist {
		indexes = &Indexes{map[string]*Index{}}
		indexCache[info.Kind] = indexes

		if info.Versioned {
			indexes.List[LastGenIndex] = &Index{
				Type: IndexTypeLastGen,
			}
		}

		t := reflect.TypeOf(info.New())
		if t.Kind() == reflect.Ptr {
			t = t.Elem()
		}
		for i := 0; i < t.NumField(); i++ {
			f := t.Field(i)
			tag := f.Tag.Get("store")

			if strings.Contains(tag, "index") {
				// todo validate that field is accessible
				transformer := info.IndexValueTransforms[f.Name]
				if transformer == nil {
					transformer = noopValueTransform
				}
				indexes.List[f.Name] = &Index{
					Type:           IndexTypeListGen,
					Field:          f.Name,
					ValueTransform: transformer,
					rFieldId:       i,
				}
			}
		}
	}

	return indexes
}

type IndexType int

const (
	IndexTypeUndef IndexType = iota
	IndexTypeLastGen
	IndexTypeListGen
)

func (indexType IndexType) String() string {
	indexTypes := [...]string{
		"lastgen",
		"listgen",
	}

	if indexType < 1 || indexType > 2 {
		panic(fmt.Sprintf("unknown index type: %d", indexType))
	}

	return indexTypes[indexType-1]
}

type Index struct {
	Type           IndexType
	Field          string
	ValueTransform runtime.ValueTransform
	rFieldId       int
}

func (index *Index) NameForStorable(storable runtime.Storable, codec Codec) string {
	key := runtime.KeyForStorable(storable)

	if index.Type == IndexTypeLastGen {
		return index.NameForValue(key, nil, codec)
	}

	t := reflect.ValueOf(storable)
	if t.Kind() == reflect.Ptr {
		t = t.Elem()
	}
	f := t.Field(index.rFieldId)

	return index.NameForValue(key, f.Interface(), codec)
}

func (index *Index) NameForValue(key runtime.Key, value interface{}, codec Codec) string {
	key = index.Type.String() + "/" + key
	if index.Type == IndexTypeLastGen {
		return key
	}

	value = index.ValueTransform(value)
	if value == nil {
		return ""
	}

	key += "/" + index.Field + "="

	if valueStr, ok := value.(string); ok {
		return key + valueStr
	}

	if valueGen, ok := value.(runtime.Generation); ok {
		return key + valueGen.String()
	}

	data, err := codec.Marshal(value)
	if err != nil {
		panic(fmt.Sprintf("error marshalling index value %s=%v", index.Field, value))
	}

	return key + string(data)
}

type IndexValueList [][]byte

func (list *IndexValueList) Add(value []byte) {
	// binary search to get desired value index in the list
	valueIndex := sort.Search(len(*list), func(index int) bool {
		return bytes.Compare((*list)[index], value) >= 0
	})

	// value already present in the list
	if valueIndex < len(*list) && bytes.Equal((*list)[valueIndex], value) {
		return
	}

	// insert value into desired position
	*list = append(*list, nil)
	copy((*list)[valueIndex+1:], (*list)[valueIndex:])
	(*list)[valueIndex] = value
}

func (list *IndexValueList) Remove(value []byte) {
	// binary search to get value index in the list
	valueIndex := sort.Search(len(*list), func(index int) bool {
		return bytes.Compare((*list)[index], value) >= 0
	})

	// remove value from the list if exists
	if valueIndex < len(*list) {
		copy((*list)[valueIndex:], (*list)[valueIndex+1:])
		(*list)[len(*list)-1] = nil
		*list = (*list)[:len(*list)-1]
	}
}

func (list *IndexValueList) Contains(value []byte) bool {
	// binary search to get value index in the list
	valueIndex := sort.Search(len(*list), func(index int) bool {
		return bytes.Compare((*list)[index], value) >= 0
	})

	return valueIndex < len(*list) && bytes.Equal((*list)[valueIndex], value)
}

func (list *IndexValueList) First() []byte {
	if len(*list) == 0 {
		return nil
	}

	return (*list)[0]
}

func (list *IndexValueList) Last() []byte {
	if len(*list) == 0 {
		return nil
	}

	return (*list)[len(*list)-1]
}
<|endoftext|>


<commit_before>package true_git

import (
	"fmt"
	"os/exec"
)

func IsAncestor(ancestorCommit, descendantCommit string, gitDir string) (bool, error) {
	gitArgs := []string{"--git-dir", gitDir, "merge-base", "--is-ancestor", ancestorCommit, descendantCommit}
	cmd := exec.Command("git", gitArgs...)

	output, err := cmd.CombinedOutput()
	if err != nil {
		if exitError, ok := err.(*exec.ExitError); ok {
			if exitError.ExitCode() == 1 {
				return false, nil
			}
		}
		return false, fmt.Errorf("'git merge-base' failed: %s:\n%s", err, output)
	}
	return true, nil
}
<commit_msg>Signatures and stages storage rework: fix merge-base error handling for 'Not a valid commit name'<commit_after>package true_git

import (
	"fmt"
	"os/exec"
	"strings"
)

func IsAncestor(ancestorCommit, descendantCommit string, gitDir string) (bool, error) {
	gitArgs := []string{"--git-dir", gitDir, "merge-base", "--is-ancestor", ancestorCommit, descendantCommit}
	cmd := exec.Command("git", gitArgs...)

	output, err := cmd.CombinedOutput()
	if err != nil {
		if exitError, ok := err.(*exec.ExitError); ok {
			if exitError.ExitCode() == 1 {
				return false, nil
			}
			if exitError.ExitCode() == 128 && strings.HasPrefix(string(output), "fatal: Not a valid commit name ") {
				return false, nil
			}
		}
		return false, fmt.Errorf("'git merge-base' failed: %s:\n%s", err, output)
	}
	return true, nil
}
<|endoftext|>